diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..fdc35d9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,18 @@ +name: CI + +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - run: python -m pip install -r requirements.txt -r requirements-mcp.txt -r requirements-dev.txt + - run: python -m pytest -q + - run: python -m compileall -q dna.py dna scripts tests + - run: python scripts/check_public_safety.py diff --git a/.github/workflows/skill-review.yml b/.github/workflows/skill-review.yml index 242a4f0..abebc87 100644 --- a/.github/workflows/skill-review.yml +++ b/.github/workflows/skill-review.yml @@ -11,12 +11,19 @@ on: jobs: review: runs-on: ubuntu-latest + env: + TESSL_API_TOKEN: ${{ secrets.TESSL_API_TOKEN }} permissions: pull-requests: write contents: read steps: - uses: actions/checkout@v4 + - name: Skip when Tessl is not configured + if: env.TESSL_API_TOKEN == '' + run: echo "Tessl Skill Review skipped because TESSL_API_TOKEN is not configured." - uses: tesslio/skill-review@main + if: env.TESSL_API_TOKEN != '' + with: + tessl-token: ${{ env.TESSL_API_TOKEN }} # Optional quality gate (off by default): - # with: # fail-threshold: 70 diff --git a/.gitignore b/.gitignore index 28f7562..bd5f5e7 100644 --- a/.gitignore +++ b/.gitignore @@ -15,8 +15,16 @@ __pycache__/ .DS_Store Thumbs.db -# 不要提交实际的记忆数据 -# memory/*.json # 如果放在项目内的话 +# Local profiles and runtime configuration +assets/config.json +docs/profiles/*.local.json -# 不要提交实际数据库 +# Never commit user memory data memory/*.db +memory/*.db-* +memory/*.json +memory/*.jsonl +!memory/README.md + +# Local safety output +.public-safety-report.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 918679f..8ff5e3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,17 +5,22 @@ All notable changes to this project will be documented in this file. ## [Unreleased] ### Added -- 新增 `QUICKSTART.md` - 5 分钟快速上手指南，聚焦核心 3 个功能 -- README.md 增加快速上手链接和核心功能说明 +- Markdown/Obsidian 长期真源与可重建 SQLite 索引 +- Codex、Claude Code、Claude Desktop 和 Hermes 的统一 stdio MCP +- 有界原生会话指针、候选提案、召回反馈与价值指标 +- 显式 `supersedes` 关系与原子回滚 +- 跨客户端共享 Skill 清单、诊断和安全同步 +- 通用 `dna-memory-loop` Skill +- CI 测试、编译和公开敏感信息扫描 ### Changed -- 优化 README.md 结构，区分核心特性和高级特性 -- 明确核心功能优先级：remember / recall / daemon +- README、快速上手和客户端文档改为当前跨端用法 +- 本机 profile、运行配置和记忆数据全部迁出 Git 仓库 +- macOS 自动化标识和示例路径改为通用命名 +- 旧 Claude 同步脚本不再包含固定用户或项目路径 -### Improved -- 降低学习成本：从 30 个脚本中明确标注核心 3 个功能 -- 提升易用性：提供快捷命令 alias 示例 -- 优化文档组织：核心功能 → 高级功能 → 实验性功能 +### Removed +- 私人 Skill、个人部署计划、真实 vault 名称和运行时记忆样例 ## [2026-04-22] - 可用性优化 @@ -31,5 +36,4 @@ All notable changes to this project will be documented in this file. --- -**维护者**: Andy / AI酋长Andy -**GitHub**: https://github.com/AIPMAndy/dna-memory +**Project**: https://github.com/AIPMAndy/dna-memory diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f7ed84e..2d257e4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,7 +40,7 @@ python3 scripts/evolve.py --help ## 联系作者 -- 微信：AIPMAndy +- 支持与讨论：请使用 GitHub Issues 或 Discussions - GitHub Issues 感谢你的贡献！🙏 diff --git a/QUICKSTART.md b/QUICKSTART.md index 5e2b744..c510ef5 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -1,257 +1,121 @@ -# 🚀 DNA Memory 快速上手指南 +# DNA Memory 快速上手 -**5 分钟让你的 AI 拥有真正的记忆！** +目标：让 Codex、Claude Code 和 Hermes 共用一个本地 Markdown/Obsidian +长期记忆库，并用 SQLite 提供可重建检索。 ---- - -## 步骤 1: 安装（30 秒） +## 1. 安装 ```bash -# 克隆到 Claude Code 技能目录 -git clone https://github.com/AIPMAndy/dna-memory.git ~/.cc-switch/skills/dna-memory +git clone https://github.com/AIPMAndy/dna-memory.git \ + "$HOME/.local/share/dna-memory/app" +cd "$HOME/.local/share/dna-memory/app" -# 进入目录 -cd ~/.cc-switch/skills/dna-memory +python3 -m venv "$HOME/.local/share/dna-memory/mcp-venv" +"$HOME/.local/share/dna-memory/mcp-venv/bin/pip" install \ + -r requirements-mcp.txt ``` -**支持的安装位置：** -- `~/.cc-switch/skills/dna-memory` ✅ -- `~/.claude/skills/dna-memory` ✅ -- `~/.openclaw/skills/dna-memory` ✅ - ---- - -## 步骤 2: 记录第一条记忆（1 分钟） +## 2. 创建仓库外 profile ```bash -# 记录一条高优先级偏好 -python3 scripts/store_memory.py \ - --content "我喜欢简洁直接的回复，不要废话" \ - --type preference \ - --weight 0.9 - -# 再记录一条技能 -python3 scripts/store_memory.py \ - --content "使用 lark-cli 操作飞书文档" \ - --type skill \ - --weight 0.7 +mkdir -p "$HOME/.config/dna-memory" "$HOME/Documents/DNA-Memory-Vault/Memory" +cp docs/profiles/profile.example.json \ + "$HOME/.config/dna-memory/profile.json" +export DNA_MEMORY_PROFILE="$HOME/.config/dna-memory/profile.json" ``` -**记忆类型：** -- `preference` - 用户偏好 -- `error` - 错误教训 -- `fact` - 事实知识 -- `skill` - 技能方法 -- `pattern` - 工作模式 -- `insight` - 洞察总结 - ---- +如果你已经有 Obsidian vault，只需把 `knowledge_root` 指向该 vault，并把 +`managed_memory_dir` 设为 DNA Memory 可管理的子目录。不要让工具接管整个 vault。 -## 步骤 3: 同步到 Claude Code（30 秒） +## 3. 初始化并检查 ```bash -# 同步高优先级记忆到 Claude Code Memory -python3 scripts/sync_to_claude.py +python3 dna.py memory reindex --json +python3 dna.py memory status --json ``` -**效果：** 权重 ≥ 0.8 的记忆会自动同步到 Claude Code Memory，**每次对话都会被加载**！ - ---- - -## 步骤 4: 查看和管理记忆（1 分钟） - -```bash -# 查看统计 -python3 dna.py manage stats - -# 列出所有记忆 -python3 dna.py manage list --limit 10 - -# 搜索记忆 -python3 dna.py manage search "简洁" - -# 查看详情 -python3 dna.py manage view 1 - -# 更新记忆权重 -python3 dna.py manage update 1 --weight 0.95 -``` - ---- - -## 步骤 5: 启用自动升华（1 分钟） - -```bash -# 查看升华状态 -python3 dna.py reflect status - -# 执行一次升华 -python3 dna.py reflect run - -# 配置自动升华（可选） -# 编辑 assets/config.json 中的 reflection 配置 -``` - -**升华机制（零 API 成本）：** -- 频繁访问的记忆 → 权重提升 -- 长期不用的记忆 → 权重衰减 -- 高权重记忆 → 晋升长期记忆 - ---- - -## 步骤 6: 监控性能（30 秒） - -```bash -# 检查性能 -python3 dna.py monitor check - -# 如果需要清理 -python3 dna.py monitor auto-clean -``` - ---- - -## 🎯 常见使用场景 +预期结果：`truth_root_exists` 为 `true`，数据库位于 profile 指定位置。 -### 场景 1: 记录用户偏好 +## 4. 配置客户端 ```bash -python3 scripts/store_memory.py \ - --content "用户是 AI 产品专家，喜欢技术细节和原理" \ - --type preference \ - --weight 0.9 -``` - -### 场景 2: 记录错误教训 +ROOT="$HOME/.local/share/dna-memory/app" +PYTHON="$HOME/.local/share/dna-memory/mcp-venv/bin/python" +PROFILE="$HOME/.config/dna-memory/profile.json" -```bash -python3 scripts/store_memory.py \ - --content "不要使用 selenium 控制浏览器，应该用 Kimi webbridge MCP 工具" \ - --type error \ - --weight 0.85 -``` +codex mcp add dna-memory \ + --env "DNA_MEMORY_PROFILE=$PROFILE" \ + -- "$PYTHON" "$ROOT/scripts/memory_mcp.py" -### 场景 3: 记录工作模式 +claude mcp add --scope user dna-memory \ + -e "DNA_MEMORY_PROFILE=$PROFILE" \ + -- "$PYTHON" "$ROOT/scripts/memory_mcp.py" -```bash -python3 scripts/store_memory.py \ - --content "复杂任务先用 /brainstorming 分析，再用 /writing-plans 规划" \ - --type pattern \ - --weight 0.75 +hermes mcp add dna-memory \ + --command "$PYTHON" \ + --env "DNA_MEMORY_PROFILE=$PROFILE" \ + --args "$ROOT/scripts/memory_mcp.py" ``` -### 场景 4: 记录技能知识 - ```bash -python3 scripts/store_memory.py \ - --content "使用 lark-doc 读取飞书文档，优先 outline/section 模式" \ - --type skill \ - --weight 0.7 +codex mcp get dna-memory +claude mcp get dna-memory +hermes mcp test dna-memory ``` ---- +Claude Desktop 的配置方式不同，见 +[统一 MCP 与客户端接入](docs/mcp-and-client-adapters.md)。 -## 💡 权重设置建议 +## 5. 做一次真实闭环验收 -| 权重范围 | 类型 | 说明 | 是否同步到 Claude Code | -|----------|------|------|------------------------| -| 0.9-1.0 | 核心偏好 | 用户最重要的偏好，绝对不能忘 | ✅ 是 | -| 0.8-0.9 | 重要规则 | 重要的工作规则、错误教训 | ✅ 是 | -| 0.6-0.8 | 常用技能 | 经常用到的技能、模式 | ❌ 否 | -| 0.4-0.6 | 一般信息 | 一般性知识、事实 | ❌ 否 | -| < 0.4 | 临时记忆 | 临时性的、可能过时的信息 | ❌ 否 | +在任一已接入客户端中： ---- +1. 调用 `memory_remember` 写入一条无敏感信息的测试结论。 +2. 用该结论中的独特关键词调用 `memory_recall`。 +3. 确认返回相同 memory ID。 +4. 调用 `memory_feedback`，`outcome` 设为 `useful`。 +5. 在另一个客户端再次召回同一关键词。 -## 🔄 自动化配置（可选） +只看到 MCP 配置不代表系统已经生效。写入、同端召回、跨端召回和反馈都成功， +才算完成验收。验收后可以用新记忆的 `supersedes` 替换测试结论，或从受管 +Markdown 目录删除它并执行 `memory reindex`。 -### 配置自动同步（每小时） - -```bash -# 加载 launchd 配置 -launchctl load ~/Library/LaunchAgents/com.andy.dna-memory-sync.plist +## 6. 安装统一行为 Skill -# 查看状态 -launchctl list | grep dna-memory - -# 卸载 -launchctl unload ~/Library/LaunchAgents/com.andy.dna-memory-sync.plist -``` - ---- - -## 📊 查看完整命令列表 +把 `skills/dna-memory-loop` 复制到共享 Skill 真源，注册后 dry-run： ```bash -python3 dna.py help -``` +mkdir -p "$HOME/.agents/skills" +cp -R skills/dna-memory-loop "$HOME/.agents/skills/" +cp assets/skills.example.json "$HOME/.config/dna-memory/skills.json" -**输出：** -``` -DNA Memory - 智能记忆管理系统 - -可用命令: - manage - 记忆管理（查看、搜索、编辑、删除） - reflect - 记忆升华（轻量级，不调用LLM） - monitor - 性能监控 - ask - 智能问答（自动注入相关记忆） - sync - 同步高优先级记忆到 Claude Code Memory +python3 dna.py skills doctor --json +python3 dna.py skills sync --json +python3 dna.py skills sync --apply --json ``` ---- - -## ❓ 常见问题 - -### Q1: 如何知道哪些记忆被同步了？ +## 7. 观察是否产生价值 ```bash -# 查看同步到 Claude Code 的记忆文件 -cat ~/.claude/projects/*/memory/synced-preferences.md +python3 dna.py memory coverage --json +python3 dna.py memory value --json ``` -### Q2: 记忆太多影响性能怎么办？ - -```bash -# 检查性能 -python3 dna.py monitor check - -# 自动清理低权重记忆 -python3 dna.py monitor auto-clean -``` - -### Q3: 如何调整升华频率？ - -编辑 `assets/config.json`： +分别看四件事：客户端是否被覆盖、长期记忆是否实际增长、召回是否命中、 +命中后是否获得 `useful` 反馈。候选事件数量很大但长期结论和有用召回不增长， +不代表系统有价值。 -```json -{ - "reflection": { - "interval_hours": 12 // 改为 12 小时一次 - } -} -``` - -### Q4: 如何删除错误的记忆？ +## 8. 维护 ```bash -# 搜索找到 ID -python3 dna.py manage search "错误关键词" - -# 删除 -python3 dna.py manage delete --confirm +python3 dna.py memory maintain daily --json +python3 dna.py memory maintain weekly --json +python3 dna.py memory maintain monthly --json ``` ---- - -## 🎉 完成！ - -现在你的 AI 有了真正的记忆系统！ - -**下一步：** -- 📚 阅读 [完整文档](./README.md) -- 🔧 查看 [性能优化指南](./PERFORMANCE_OPTIMIZATION.md) -- 🤝 参与 [贡献](./CONTRIBUTING.md) - ---- +- daily：审查并结晶安全提案，清理过期候选。 +- weekly：daily + SQLite 备份与压缩。 +- monthly：weekly + 完整性检查与 Markdown 重建核对。 -**遇到问题？** [提交 Issue](https://github.com/AIPMAndy/dna-memory/issues) +完整会话仍属于各客户端。DNA Memory 不应复制完整 transcript，也不应保存 +token、API key、私钥或账号配置。 diff --git a/README.md b/README.md index dcdc95a..86a2e75 100644 --- a/README.md +++ b/README.md @@ -1,348 +1,234 @@
-# 🧬 DNA Memory +# DNA Memory -**让 AI Agent 像人脑一样学习、强化、遗忘与进化** +**面向 Codex、Claude Code、Claude Desktop、Hermes 与 Obsidian 的本地优先统一记忆层** [![Stars](https://img.shields.io/github/stars/AIPMAndy/dna-memory?style=social)](https://github.com/AIPMAndy/dna-memory/stargazers) [![License](https://img.shields.io/github/license/AIPMAndy/dna-memory)](https://github.com/AIPMAndy/dna-memory) -[![Python](https://img.shields.io/badge/Python-3.8+-blue)](https://www.python.org/) -[![Version](https://img.shields.io/badge/version-3.2-green)](https://github.com/AIPMAndy/dna-memory/releases) +[![Python](https://img.shields.io/badge/Python-3.9+-blue)](https://www.python.org/) -[English](./README_EN.md) | **简体中文** | [快速上手](./QUICKSTART.md) +[English](./README_EN.md) | 简体中文 | [快速上手](./QUICKSTART.md)
---- +DNA Memory 把一个 Markdown/Obsidian 目录作为长期记忆真源，把 SQLite +作为可删除、可重建的索引。多个 AI 客户端通过同一个 MCP 服务召回、写入和反馈， +不再各自维护互相冲突的长期记忆副本。 -> **大多数 AI 记忆系统只是在"存储"。** -> **DNA Memory 解决的是: AI 如何像人一样学习、强化与进化。** +它不会把“保存所有聊天记录”等同于“形成记忆”。完整会话仍由原客户端保管； +DNA Memory 只保存有界来源指针、经过审查的提案，以及验证后的关键结论。 -## 💡 为什么需要 DNA Memory？ +## 核心边界 -你是否遇到过这些问题： +- Markdown 是长期真源，SQLite 是索引和遥测层。 +- `memory_remember` 写入的是短小、可复用结论，不是 transcript。 +- 自动导入默认只保存 session ID、路径、哈希、偏移和计数。 +- 自动提炼每个会话最多产生 3 条候选，每条最多 800 字符。 +- 凭证、私钥、常见 token 和疑似敏感内容会被拒绝。 +- 普通 Claude Desktop 云端聊天没有稳定本地正文来源时，只能显式 MCP 写回。 +- 任何客户端的记忆故障都不应阻塞主任务。 -- ❌ **AI 总是忘记你的偏好**：每次都要重复"我喜欢简洁的回复" -- ❌ **犯过的错误反复出现**：上次说过不要用某个工具，这次又推荐 -- ❌ **记忆混乱无序**：存了一堆碎片，检索时找不到关键信息 -- ❌ **性能问题**：记忆越多越卡，最后弃用 +## 架构 -**DNA Memory 三大核心价值：** - -1. **🎯 真正有效的记忆** - 高优先级记忆自动同步到 Claude Code，每次对话都会被加载 -2. **⚡️ 轻量高性能** - 500 条记忆 < 0.1MB，查询 < 0.5ms，不影响 AI 使用体验 -3. **🧠 自动升华** - 不调用 LLM，纯算法实现记忆强化、衰减、晋升，零成本 - ---- - -## 🚀 5 分钟快速上手 - -```bash -# 1. 克隆到技能目录 -git clone https://github.com/AIPMAndy/dna-memory.git ~/.cc-switch/skills/dna-memory -# 或 ~/.claude/skills/dna-memory - -cd ~/.cc-switch/skills/dna-memory - -# 2. 记录一条高优先级偏好 -python3 scripts/store_memory.py \ - --content "浏览器操作永远优先使用 Kimi webbridge MCP 工具" \ - --type preference \ - --weight 0.95 - -# 3. 同步到 Claude Code Memory（每次对话都会加载） -python3 scripts/sync_to_claude.py - -# 4. 查看统计 -python3 dna.py manage stats - -# 5. 搜索记忆 -python3 dna.py manage search "浏览器" +```text +Codex / Claude / Hermes + | + | stdio MCP + v + memory_recall / remember / feedback + | + +--> Markdown vault <- durable source of truth + | + +--> SQLite index <- rebuildable search + telemetry + | + +--> bounded candidates <- pointers and reviewable proposals + +Native client histories remain in their original stores. ``` -**完成！** 你的 AI 现在会记住这条偏好。 - ---- - -## ✨ 核心能力 - -### 1. 📊 记忆管理界面 +## 安装 ```bash -# 查看所有记忆 -python3 dna.py manage list --limit 10 +git clone https://github.com/AIPMAndy/dna-memory.git \ + "$HOME/.local/share/dna-memory/app" +cd "$HOME/.local/share/dna-memory/app" -# 搜索记忆（FTS5 全文搜索） -python3 dna.py manage search "webbridge" +python3 -m venv "$HOME/.local/share/dna-memory/mcp-venv" +"$HOME/.local/share/dna-memory/mcp-venv/bin/pip" install \ + -r requirements-mcp.txt -# 查看详情 -python3 dna.py manage view 9 - -# 更新记忆 -python3 dna.py manage update 9 --weight 1.0 --type preference +mkdir -p "$HOME/.config/dna-memory" "$HOME/Documents/DNA-Memory-Vault/Memory" +cp docs/profiles/profile.example.json \ + "$HOME/.config/dna-memory/profile.json" +``` -# 删除记忆 -python3 dna.py manage delete 9 --confirm +按需编辑仓库外的 profile： -# 统计信息 -python3 dna.py manage stats +```json +{ + "knowledge_root": "~/Documents/DNA-Memory-Vault", + "database_path": "~/.local/share/dna-memory/memory.db", + "managed_memory_dir": "Memory", + "skill_root": "~/.agents/skills", + "skill_registry": "~/.config/dna-memory/skills.json", + "platform_skill_roots": { + "codex": "~/.codex/skills", + "claude": "~/.claude/skills", + "hermes": "~/.hermes/skills" + } +} ``` -**输出示例：** -``` -📊 DNA Memory 统计信息 -============================================================ -总记忆数: 10 -高优先级记忆 (≥0.8): 1 -数据库大小: 0.11 MB - -按类型分布: - preference : 2 条 (平均权重: 0.74) - pattern : 4 条 (平均权重: 0.52) - fact : 2 条 (平均权重: 0.56) - error : 1 条 (平均权重: 0.52) +```bash +export DNA_MEMORY_PROFILE="$HOME/.config/dna-memory/profile.json" +python3 dna.py memory status --json +python3 dna.py memory reindex --json ``` -### 2. 🧠 轻量级记忆升华 +## 接入三个客户端 -**零成本、纯算法、可配置频率** +以下命令让 Codex、Claude Code 和 Hermes 指向同一个 profile 与 MCP 服务： ```bash -# 查看升华状态 -python3 dna.py reflect status - -# 执行升华 -python3 dna.py reflect run - -# 强制执行（忽略时间间隔） -python3 dna.py reflect run --force - -# 查看配置 -python3 dna.py reflect config +ROOT="$HOME/.local/share/dna-memory/app" +PYTHON="$HOME/.local/share/dna-memory/mcp-venv/bin/python" +PROFILE="$HOME/.config/dna-memory/profile.json" + +codex mcp add dna-memory \ + --env "DNA_MEMORY_PROFILE=$PROFILE" \ + -- "$PYTHON" "$ROOT/scripts/memory_mcp.py" + +claude mcp add --scope user dna-memory \ + -e "DNA_MEMORY_PROFILE=$PROFILE" \ + -- "$PYTHON" "$ROOT/scripts/memory_mcp.py" + +hermes mcp add dna-memory \ + --command "$PYTHON" \ + --env "DNA_MEMORY_PROFILE=$PROFILE" \ + --args "$ROOT/scripts/memory_mcp.py" ``` -**升华机制：** -- ✅ **频繁访问 → 权重提升**：每次调用 +0.05 -- ✅ **长期不用 → 权重衰减**：每天 -0.01 -- ✅ **晋升长期记忆**：权重 ≥ 0.8 且访问 ≥ 3 次 -- ✅ **发现相似记忆**：提示合并建议（不自动执行） - -**完全不调用 LLM，零 API 成本！** - -### 3. ⚡️ 性能监控 +验收时不要只看配置文件： ```bash -# 检查性能 -python3 dna.py monitor check - -# 自动清理（性能危险时） -python3 dna.py monitor auto-clean +codex mcp get dna-memory +claude mcp get dna-memory +hermes mcp test dna-memory ``` -**性能保证：** -``` -🔍 DNA Memory 性能报告 -================================================== -数据库大小: 0.11 MB / 5 MB -记忆数量: 10 / 500 -查询速度: 0.34 ms -健康度: 98% -状态: HEALTHY - -✅ 性能良好 -``` +Claude Desktop 使用 `mcpServers` 配置，且 JSON 中必须是绝对路径。安全迁移、 +备份和回滚见 [客户端接入文档](docs/mcp-and-client-adapters.md)。 -**性能限制：** -- 最大记忆数：500 条（可配置，默认降低以保持轻量） -- 数据库上限：5 MB -- 查询时间：< 50ms -- 自动清理：权重 < 0.25 且超过 60 天未访问 +## MCP 工具 -### 4. 🔄 自动同步到 Claude Code Memory +| 工具 | 用途 | +|---|---| +| `memory_recall` | 用 1 至 4 个独立关键词召回活跃记忆 | +| `memory_get` | 按稳定 ID 获取单条记忆和替代关系 | +| `memory_remember` | 写入验证后的长期结论 | +| `memory_feedback` | 标记召回结果 `useful` 或 `misleading` | +| `memory_close_session` | 保存有界会话来源指针 | +| `memory_status` | 查看真源和索引状态 | +| `memory_reindex` | 从 Markdown 重建 SQLite 索引 | -高优先级记忆（weight ≥ 0.8）会自动同步到 Claude Code Memory，**每次对话都会被加载**！ +推荐将 [dna-memory-loop](skills/dna-memory-loop/SKILL.md) 分发到各客户端。 +它规定了同一套行为：任务前召回、确实使用后反馈、验证后才写回。 -```bash -# 手动同步 -python3 dna.py sync - -# 或通过 launchd 自动同步（每小时） -launchctl load ~/Library/LaunchAgents/com.andy.dna-memory-sync.plist -``` - -### 5. 🤖 智能问答（Memory-Enhanced Q&A） - -AI 回答时自动注入相关记忆，让 AI 更懂你： +## 日常操作 ```bash -# 使用默认 Agent -python3 dna.py ask "浏览器操作应该用什么工具？" +# 真源、容量与索引状态 +python3 dna.py memory status --json -# 指定 Agent -python3 dna.py ask "如何优化性能？" --agent hermes +# 客户端来源、自动捕获和 MCP 边界 +python3 dna.py memory coverage --json -# 调整检索数量 -python3 dna.py ask "调试技巧" --recall-limit 10 -``` +# 召回、命中、反馈、写回和积压指标 +python3 dna.py memory value --json ---- +# 从 Markdown 重建索引 +python3 dna.py memory reindex --json -## 🎯 三层记忆架构 - -```text -工作记忆 (Working Memory) - ↓ 筛选 -短期记忆 (Short-term Memory) - ↓ 巩固 / 晋升 -长期记忆 (Long-term Memory) +# 结晶安全提案、清理候选、备份、完整性检查 +python3 dna.py memory maintain daily --json +python3 dna.py memory maintain weekly --json +python3 dna.py memory maintain monthly --json ``` -| 层级 | 作用 | 典型内容 | 权重范围 | -|------|------|----------|----------| -| 工作记忆 | 当前会话临时上下文 | 本轮任务、刚发生的事 | 0.3-0.5 | -| 短期记忆 | 近期重要信息 | 用户偏好、近期经验、错误教训 | 0.5-0.8 | -| 长期记忆 | 稳定知识与模式 | 规则、技能、长期偏好、归纳模式 | 0.8-1.0 | - ---- - -## 📦 记忆类型 - -| 类型 | 说明 | 示例 | 推荐权重 | -|------|------|------|----------| -| `preference` | 用户偏好、习惯 | "回复要简洁直接" | 0.8-1.0 | -| `pattern` | 工作模式、流程 | "马斯克五步法" | 0.6-0.9 | -| `skill` | 技能、工具使用 | "如何使用 lark-cli" | 0.5-0.8 | -| `error` | 错误教训 | "不要用 selenium，用 webbridge" | 0.7-0.9 | -| `fact` | 事实性信息 | "用户是 AI 产品专家" | 0.5-0.7 | -| `insight` | 洞察、总结 | "内容质量不是问题，缺付费入口" | 0.6-0.9 | - ---- - -## ⚙️ 配置文件 - -编辑 `assets/config.json`： +当新证据明确使旧结论失效时，调用 `memory_remember` 并显式传入旧 ID： ```json { - "_comment_performance": "性能配置", - "max_total_memories": 500, // 最大记忆数（降低保持轻量） - "cleanup_threshold": 0.25, // 清理阈值 - "archive_after_days": 90, // 归档天数 - - "_comment_sync": "同步配置", - "sync_weight_threshold": 0.8, // 同步权重阈值 - "sync_max_memories": 20, // 最多同步数 - - "_comment_reflection": "升华配置（零成本）", - "reflection": { - "enabled": true, // 启用升华 - "interval_hours": 24, // 升华频率（小时） - "min_memories_for_reflection": 5, // 最少记忆数 - "weight_boost_per_recall": 0.05, // 每次调用增加权重 - "weight_decay_per_day": 0.01 // 每天衰减权重 - } + "type": "project_state", + "summary": "客户端接入已经通过真实召回与写回验收。", + "supersedes": ["mem_old_unverified"] } ``` ---- - -## 🛠️ 安装位置灵活 - -DNA Memory 自动检测安装位置，支持： - -- ✅ `~/.cc-switch/skills/dna-memory` -- ✅ `~/.claude/skills/dna-memory` -- ✅ `~/.openclaw/skills/dna-memory` - -无需修改代码，自动适配！ - ---- - -## 🌟 项目特色 - -### 1. 真正轻量 -- **核心只依赖 Python 标准库 + SQLite** -- **500 条记忆 < 0.1MB** -- **查询 < 0.5ms** -- **不影响 AI 使用性能** - -### 2. 零 API 成本 -- **记忆升华完全不调用 LLM** -- **纯算法实现强化、衰减、晋升** -- **长期运行零额外成本** - -### 3. 真正有效 -- **高优先级记忆自动同步到 Claude Code Memory** -- **每次对话都会被加载** -- **AI 真的会记住你的偏好** +旧 Markdown 会保留并改为 `superseded`；默认召回只返回 active 结论。 +系统不会仅按项目或类型猜测冲突。 -### 4. 可管理 -- **完整的命令行界面** -- **查看、搜索、编辑、删除** -- **统计、监控、升华** -- **完全掌控你的记忆** +## 自动导入与提炼 ---- - -## 📚 多 Agent 支持 - -DNA Memory 支持多个 AI Agent： - -- ✅ **Claude CLI** (v2.1.181+) -- ✅ **Hermes** (v0.17.0+) -- ⚠️ **Codex** (需要重新安装) - -自动检测可用 Agent，统一调用接口。 - ---- - -## 🔧 开发指南 +可按需要运行： ```bash -# 克隆仓库 -git clone https://github.com/AIPMAndy/dna-memory.git -cd dna-memory - -# 运行测试 -python3 scripts/memory_manager.py stats -python3 scripts/lightweight_monitor.py check -python3 scripts/lightweight_reflection.py status - -# 查看帮助 -python3 dna.py help +python3 scripts/import_codex_rollouts.py +python3 scripts/import_claudian_sessions.py +python3 scripts/import_claude_desktop_sessions.py +python3 scripts/import_hermes_sessions.py +python3 scripts/import_native_history.py ``` ---- - -## 📄 许可证 +导入器使用检查点和幂等事件 ID。普通事件只保存指针；只有显式 +`DNA_MEMORY_PROPOSAL {JSON}` 或通过有界信号提取的短结论才进入候选队列， +并且仍需经过 daily 维护的类型、敏感信息、容量和去重检查。 -MIT License - 详见 [LICENSE](./LICENSE) +这意味着“扫描到了会话”不等于“已经形成长期认知”。衡量系统价值时应同时看： ---- +1. 自动捕获覆盖率。 +2. 长期记忆写入数量。 +3. 召回命中率与 `useful` 反馈。 +4. 是否减少重复说明和重复错误。 -## 🤝 贡献 +## 跨客户端 Skill 管理 -欢迎 PR、Issue、建议！ +共享 Skill 真源与记忆真源分开管理。注册表只声明 DNA Memory 有权分发的 +Skill，注册表外的客户端专属目录不会被删除或覆盖。 -详见 [CONTRIBUTING.md](./CONTRIBUTING.md) +```bash +cp assets/skills.example.json "$HOME/.config/dna-memory/skills.json" +python3 dna.py skills inventory --json +python3 dna.py skills doctor --json +python3 dna.py skills sync --json +python3 dna.py skills sync --apply --json +``` ---- +`sync` 默认 dry-run，`--apply` 只创建缺失符号链接。详见 +[Skill 管理](docs/skill-management.md)。 -## 🙏 致谢 +## 隐私与公开安全 -灵感来源： -- 人类记忆的工作原理 -- Ebbinghaus 遗忘曲线 -- 强化学习理论 +仓库不包含真实 profile、数据库、会话、记忆 JSON、个人 Skill 或内部部署记录。 +发布前运行： ---- +```bash +python3 scripts/check_public_safety.py +``` -
+本地 profile、备份、SQLite 和 Markdown vault 不应位于 Git 仓库内。 +如果曾误提交凭证，仅删除文件不够，还必须撤销凭证并清理 Git 历史。 -**如果这个项目对你有帮助，请给一个 ⭐️ Star！** +## 开发验证 -[GitHub](https://github.com/AIPMAndy/dna-memory) | [文档](./docs) | [快速上手](./QUICKSTART.md) +```bash +python3 -m pytest -q +python3 -m compileall -q dna.py dna scripts tests +python3 scripts/check_public_safety.py +git diff --check +``` -Made with ❤️ by [Andy](https://github.com/AIPMAndy) +## License -
+[MIT](LICENSE) diff --git a/README_EN.md b/README_EN.md index 8b8a84a..79a7b49 100644 --- a/README_EN.md +++ b/README_EN.md @@ -1,321 +1,183 @@
-# 🧬 DNA Memory +# DNA Memory -**Make AI Agents learn, reinforce, forget, and generalize like a human brain** +**A local-first shared memory layer for Codex, Claude, Hermes, and Obsidian** [![Stars](https://img.shields.io/github/stars/AIPMAndy/dna-memory?style=social)](https://github.com/AIPMAndy/dna-memory/stargazers) [![License](https://img.shields.io/github/license/AIPMAndy/dna-memory)](https://github.com/AIPMAndy/dna-memory) -[![Python](https://img.shields.io/badge/Python-3.8+-blue)](https://www.python.org/) -[![OpenClaw](https://img.shields.io/badge/Built%20for-OpenClaw-purple)](https://github.com/openclaw/openclaw) +[![Python](https://img.shields.io/badge/Python-3.9+-blue)](https://www.python.org/) -**English** | [简体中文](./README.md) +English | [简体中文](./README.md) | [Quick start](./QUICKSTART.md)
---- +DNA Memory uses a Markdown/Obsidian directory as the durable source of truth +and SQLite as a disposable, rebuildable index. Codex, Claude Code, Claude +Desktop, and Hermes can use the same stdio MCP server instead of keeping +conflicting long-term memory copies. -> Most AI memory systems only solve **storage**. -> **DNA Memory** is about how agents actually **learn and evolve**. +It does not equate storing every conversation with learning. Native transcripts +stay in their original clients. DNA Memory stores bounded provenance pointers, +reviewable proposals, and verified reusable conclusions. -It is not just a memory store. It is a memory evolution system with: -- **3-layer memory architecture** -- **reinforcement and decay** -- **reflection (`reflect`)** -- **promotion to long-term memory (`promote`)** -- **duplicate cleanup (`dedupe`)** -- **FTS5-powered recall search** -- **background daemon maintenance** +## Safety model ---- +- Markdown is durable truth; SQLite is search, queue, and telemetry state. +- `memory_remember` stores concise conclusions, not transcripts. +- Native importers retain IDs, paths, hashes, offsets, and counts by default. +- Automatic extraction produces at most three candidates per session and 800 + characters per summary. +- Credential-like content is rejected. +- Cloud-only Claude Desktop chats require explicit MCP writeback when no stable + local transcript source is available. +- Memory failure never blocks the primary task. -## 🆚 Why not just use a normal memory store? - -| Capability | Mem0 | Zep | LangChain Memory | **DNA Memory** | -|------------|:----:|:---:|:----------------:|:--------------:| -| Basic storage | ✅ | ✅ | ✅ | ✅ | -| Vector / semantic retrieval | ✅ | ✅ | ✅ | ⚠️ extensible | -| Multi-layer architecture | ❌ | ⚠️ | ❌ | ✅ **working / short / long** | -| Active forgetting | ❌ | ❌ | ❌ | ✅ | -| Reflection loop | ❌ | ❌ | ❌ | ✅ | -| Pattern extraction | ❌ | ❌ | ❌ | ✅ | -| Long-term promotion | ❌ | ❌ | ❌ | ✅ | -| Local-first / minimal core deps | ❌ | ❌ | ❌ | ✅ | -| Built for agent workflows | ⚠️ | ⚠️ | ⚠️ | ✅ | - -**Positioning in one sentence:** - -> DNA Memory helps AI agents not only remember, but also reinforce, forget, summarize, and evolve like a real cognitive system. - ---- - -## 🚀 Quick Start in 30 Seconds +## Install ```bash -# 1) Clone into your OpenClaw skills directory -git clone https://github.com/AIPMAndy/dna-memory.git ~/.openclaw/skills/dna-memory - -# 2) Remember one preference -python3 ~/.openclaw/skills/dna-memory/scripts/evolve.py remember "The user prefers concise and direct responses" -t preference -i 0.9 +git clone https://github.com/AIPMAndy/dna-memory.git \ + "$HOME/.local/share/dna-memory/app" +cd "$HOME/.local/share/dna-memory/app" -# 3) Recall related memories -python3 ~/.openclaw/skills/dna-memory/scripts/evolve.py recall "concise direct" +python3 -m venv "$HOME/.local/share/dna-memory/mcp-venv" +"$HOME/.local/share/dna-memory/mcp-venv/bin/pip" install \ + -r requirements-mcp.txt -# 4) Inspect stats -python3 ~/.openclaw/skills/dna-memory/scripts/evolve.py stats +mkdir -p "$HOME/.config/dna-memory" "$HOME/Documents/DNA-Memory-Vault/Memory" +cp docs/profiles/profile.example.json \ + "$HOME/.config/dna-memory/profile.json" ``` -**Why it is practical:** -- core features run on Python + SQLite -- no external database required -- local-first by default -- ideal for personal assistants, local agents, and autonomous workflows +The profile stays outside the repository. Adjust the vault and platform Skill +roots, then verify the store: ---- - -## ✨ Core Capabilities - -### 1. Three-layer memory architecture - -```text -Working Memory - ↓ filter -Short-term Memory - ↓ consolidate / promote -Long-term Memory +```bash +export DNA_MEMORY_PROFILE="$HOME/.config/dna-memory/profile.json" +python3 dna.py memory status --json +python3 dna.py memory reindex --json ``` -| Layer | Role | Typical content | -|------|------|-----------------| -| Working | temporary session context | current task state, fresh facts | -| Short-term | recent important information | preferences, lessons, recent errors | -| Long-term | stable knowledge and patterns | rules, skills, persistent preferences | - -### 2. Reinforcement and forgetting - -- **used often → higher weight** -- **unused for a long time → decay** -- **low-weight memories → removable** -- **stable high-value memories → promoted to long-term memory** +## Connect Codex, Claude Code, and Hermes -### 3. Reflection (`reflect`) - -`reflect` does two things: -- extracts recurring patterns from recent high-weight memories -- promotes stable short-term memories into long-term memory - -### 4. Better recall search - -Current recall supports: -- **multi-keyword AND search** -- **type filters** like `type:error` / `type:skill` -- **SQLite FTS5 full-text search** -- automatic fallback to LIKE search if FTS5 is unavailable +```bash +ROOT="$HOME/.local/share/dna-memory/app" +PYTHON="$HOME/.local/share/dna-memory/mcp-venv/bin/python" +PROFILE="$HOME/.config/dna-memory/profile.json" + +codex mcp add dna-memory \ + --env "DNA_MEMORY_PROFILE=$PROFILE" \ + -- "$PYTHON" "$ROOT/scripts/memory_mcp.py" + +claude mcp add --scope user dna-memory \ + -e "DNA_MEMORY_PROFILE=$PROFILE" \ + -- "$PYTHON" "$ROOT/scripts/memory_mcp.py" + +hermes mcp add dna-memory \ + --command "$PYTHON" \ + --env "DNA_MEMORY_PROFILE=$PROFILE" \ + --args "$ROOT/scripts/memory_mcp.py" +``` -Examples: +Verify runtime connectivity, not only configuration text: ```bash -python3 scripts/evolve.py recall "feishu api" -python3 scripts/evolve.py recall "type:error github" -python3 scripts/evolve.py recall "user preference concise" +codex mcp get dna-memory +claude mcp get dna-memory +hermes mcp test dna-memory ``` -### 5. Background maintenance daemon - -The daemon can automatically run: -- `reflect` -- `decay` -- throttled maintenance so the same batch is not repeatedly summarized - -It can also be registered with **macOS launchd** for auto-start on boot. - ---- - -## 📦 Actual current architecture - -```text -dna-memory/ -├── scripts/ -│ ├── evolve.py # core CLI: remember / recall / stats / reflect / dedupe ... -│ ├── dna_memory_daemon.py # background maintenance daemon -│ ├── semantic_search.py # experimental semantic search module -│ ├── analyze.py -│ ├── api.py -│ ├── autocollect.py -│ ├── backup.py -│ ├── cli.py -│ ├── detailed_stats.py -│ ├── knowme_link.py -│ ├── reminder.py -│ ├── trigger.py -│ └── visualize.py -├── memory/ -│ ├── memory.db # SQLite primary store (memories + operations) -│ └── working.json # working memory -├── assets/ -│ └── config.json # daemon / decay config -├── README.md -├── README_EN.md -└── SKILL.md -``` +Claude Desktop uses `mcpServers` with absolute paths. See +[client setup](docs/mcp-and-client-adapters.md) for migration, backup, hooks, +importers, and rollback. -> Note: `memory/*.db` should not be committed. The repo now ignores real memory database files by default. +## MCP tools ---- +| Tool | Purpose | +|---|---| +| `memory_recall` | Recall active memories for a focused query | +| `memory_get` | Fetch one memory and its replacement relationships | +| `memory_remember` | Write a verified durable conclusion | +| `memory_feedback` | Record `useful` or `misleading` recall feedback | +| `memory_close_session` | Store bounded session provenance | +| `memory_status` | Inspect truth and index state | +| `memory_reindex` | Rebuild SQLite from Markdown | -## 🧪 Core Commands +Distribute the bundled [dna-memory-loop](skills/dna-memory-loop/SKILL.md) to +each client. It establishes the same behavior everywhere: recall before +context-dependent work, give feedback only for used results, and write back +only after verification. -### Remember +## Operations ```bash -python3 scripts/evolve.py remember "Andy prefers concise and direct responses" -t preference -i 0.95 +python3 dna.py memory status --json +python3 dna.py memory coverage --json +python3 dna.py memory value --json +python3 dna.py memory reindex --json +python3 dna.py memory maintain daily --json +python3 dna.py memory maintain weekly --json +python3 dna.py memory maintain monthly --json ``` -### Recall +When a new verified fact invalidates an older one, pass exact old memory IDs in +`supersedes`. Old Markdown remains available as history, while default recall +returns active conclusions only. DNA Memory never infers replacement from type +or project alone. -```bash -python3 scripts/evolve.py recall "concise response" -python3 scripts/evolve.py recall "type:skill feishu" -``` - -### Stats +## Bounded native import ```bash -python3 scripts/evolve.py stats +python3 scripts/import_codex_rollouts.py +python3 scripts/import_claudian_sessions.py +python3 scripts/import_claude_desktop_sessions.py +python3 scripts/import_hermes_sessions.py +python3 scripts/import_native_history.py ``` -### Reflect +Importers are incremental and idempotent. Ordinary events remain provenance +pointers. Explicit `DNA_MEMORY_PROPOSAL {JSON}` markers and bounded signal +extraction create review candidates; daily maintenance still applies type, +sensitivity, capacity, and deduplication gates. -```bash -python3 scripts/evolve.py reflect -``` +Capture is not the same as durable learning. Evaluate both capture coverage and +the number of verified memories that are later recalled and marked useful. -### Promote +## Shared Skill management ```bash -python3 scripts/evolve.py promote --id 12 +cp assets/skills.example.json "$HOME/.config/dna-memory/skills.json" +python3 dna.py skills inventory --json +python3 dna.py skills doctor --json +python3 dna.py skills sync --json +python3 dna.py skills sync --apply --json ``` -### Dedupe +The registry grants distribution authority only for named Skills. Unregistered +platform-specific Skills are never deleted or overwritten. `sync` is a dry run +unless `--apply` is present. -```bash -python3 scripts/evolve.py dedupe -``` +## Public release safety -### Daemon +Runtime profiles, databases, native sessions, memory JSON, private Skills, and +deployment notes are excluded from source control. Run this before publishing: ```bash -# start -python3 scripts/dna_memory_daemon.py start - -# check status -python3 scripts/dna_memory_daemon.py status - -# stop -python3 scripts/dna_memory_daemon.py stop +python3 scripts/check_public_safety.py ``` ---- - -## ⚙️ Use Cases - -### 1. Personal AI assistants -- remember user preferences -- develop a stable collaboration style over time -- learn from mistakes instead of repeating them - -### 2. Agent workflow orchestration -- turn finished tasks into reusable skills -- store failure cases as error memories -- extract patterns from long-running work - -### 3. AI products with personalization -- accumulate user profiles -- track behavioral patterns -- build long-term personalization - -### 4. Self-improving agent systems -- works well with OpenClaw, self-improving-agent, and custom agent stacks -- turns operational experience into reusable memory assets - ---- - -## 🧭 Recommended Workflow - -```text -Receive task - ↓ -Recall related memories - ↓ -Execute - ↓ -Remember new preferences / skills / errors - ↓ -Reflect recurring patterns - ↓ -Promote into long-term memory -``` - -This workflow is especially useful when: -- the user corrects the agent -- a new preference is learned -- an API/tool fails -- a long task finishes -- a repeatable pattern appears - ---- - -## 🗺️ Roadmap - -- [x] SQLite single-store refactor -- [x] remember / recall / reflect / promote / dedupe CLI -- [x] daemon for automatic reflect / decay -- [x] FTS5-based recall search -- [x] launchd auto-start setup -- [ ] better Chinese tokenization and ranking -- [ ] real embedding-based semantic retrieval -- [ ] stronger memory graph visualization -- [ ] more complete import / export / migration tooling -- [ ] shared memory spaces for multi-agent systems - ---- - -## 🤝 Contributing - -Issues and PRs are welcome. - -High-impact contribution areas: -- recall ranking quality -- Chinese search experience -- pattern extraction quality -- memory visualization -- embedding provider integrations +If a credential was ever committed, deleting the current file is insufficient: +revoke the credential and clean the Git history. ---- +## Validate -## 👨‍💻 Author - -**Andy / AI酋长Andy** -Ex-Tencent / Baidu AI Product Expert → LLM Unicorn VP → Startup CEO - -Focus areas: -- AI agents -- AI commercialization -- memory systems -- human augmentation - -GitHub: https://github.com/AIPMAndy - ---- - -## 📄 License - -[Apache 2.0](LICENSE) - ---- - -
+```bash +python3 -m pytest -q +python3 -m compileall -q dna.py dna scripts tests +python3 scripts/check_public_safety.py +git diff --check +``` -**If this project helps you, give it a ⭐ Star.** +## License -
+[MIT](LICENSE) diff --git a/assets/config.example.json b/assets/config.example.json new file mode 100644 index 0000000..63604ad --- /dev/null +++ b/assets/config.example.json @@ -0,0 +1,20 @@ +{ + "knowledge_root": "~/Documents/DNA-Memory-Vault", + "database_path": "~/.local/share/dna-memory/memory.db", + "skill_root": "~/.agents/skills", + "skill_registry": "~/.config/dna-memory/skills.json", + "platform_skill_roots": { + "codex": "~/.codex/skills", + "claude": "~/.claude/skills", + "hermes": "~/.hermes/skills" + }, + "warning_bytes": 104857600, + "hard_bytes": 262144000, + "max_records": 10000, + "max_candidate_events": 10000, + "backup_dir": "~/.local/share/dna-memory/backups/managed", + "backup_keep": 8, + "claudian_session_dirs": [], + "claude_desktop_session_dirs": [], + "hermes_state_db": null +} diff --git a/assets/config.json b/assets/config.json deleted file mode 100644 index 025d18e..0000000 --- a/assets/config.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "decay_days": 7, - "decay_rate": 0.1, - "forget_threshold": 0.2, - "reflect_trigger": 20, - "max_short_term": 100, - "max_long_term": 500, - "embedding_model": "text-embedding-3-small", - "auto_reflect": true, - "auto_reflect_interval_minutes": 30, - "auto_decay": true, - "auto_decay_interval_hours": 24, - - "_comment_performance": "性能优化配置", - "max_total_memories": 500, - "cleanup_threshold": 0.25, - "archive_after_days": 90, - "auto_cleanup": true, - "auto_cleanup_interval_hours": 168, - - "_comment_sync": "同步配置", - "sync_weight_threshold": 0.8, - "sync_max_memories": 20, - "sync_prefer_recent": true, - - "_comment_reflection": "轻量级升华配置（不调用LLM，纯算法）", - "reflection": { - "enabled": true, - "interval_hours": 24, - "min_memories_for_reflection": 5, - "weight_boost_per_recall": 0.05, - "weight_decay_per_day": 0.01, - "similar_merge_threshold": 0.7 - } -} diff --git a/assets/skills.example.json b/assets/skills.example.json new file mode 100644 index 0000000..72d6f4e --- /dev/null +++ b/assets/skills.example.json @@ -0,0 +1,7 @@ +{ + "skills": { + "dna-memory-loop": { + "targets": ["codex", "claude", "hermes"] + } + } +} diff --git a/dna b/dna index 511e9f5..82b3dcc 100755 --- a/dna +++ b/dna @@ -8,9 +8,10 @@ import sys import subprocess from pathlib import Path -DNA_MEMORY_DIR = Path.home() / ".cc-switch/skills/dna-memory" +DNA_MEMORY_DIR = Path(__file__).resolve().parent EVOLVE_SCRIPT = DNA_MEMORY_DIR / "scripts/evolve.py" SYNC_SCRIPT = DNA_MEMORY_DIR / "scripts/sync_to_claude.py" +MAIN_CLI = DNA_MEMORY_DIR / "dna.py" def main(): if len(sys.argv) < 2: @@ -24,6 +25,10 @@ def main(): command = sys.argv[1] + if command in {"memory", "skills", "manage", "monitor", "ask"}: + result = subprocess.run(["python3", str(MAIN_CLI)] + sys.argv[1:]) + sys.exit(result.returncode) + if command == "remember": # 记录记忆 args = [str(EVOLVE_SCRIPT), "remember"] + sys.argv[2:] diff --git a/dna.py b/dna.py index 9c1aece..871c8f1 100755 --- a/dna.py +++ b/dna.py @@ -55,6 +55,25 @@ 'dna sync', ] }, + 'memory': { + 'script': 'memory_cli.py', + 'description': '统一长期记忆索引（状态、重建、维护）', + 'examples': [ + 'dna memory status --json', + 'dna memory reindex --json', + 'dna memory maintain daily --json', + 'dna memory maintain weekly --json', + 'dna memory maintain monthly --json', + ] + }, + 'skills': { + 'script': 'skills_cli.py', + 'description': '跨客户端 Skill 管理（清单、诊断、同步）', + 'examples': [ + 'dna skills inventory --json', + 'dna skills sync --json', + ] + }, } @@ -106,8 +125,9 @@ def main(): try: result = subprocess.run( - ['python3', str(script_path)] + args, - check=False + ['python3', '-m', 'scripts.' + script_path.stem] + args, + check=False, + cwd=str(Path(__file__).parent), ) sys.exit(result.returncode) except KeyboardInterrupt: diff --git a/docs/mcp-and-client-adapters.md b/docs/mcp-and-client-adapters.md new file mode 100644 index 0000000..3d7418f --- /dev/null +++ b/docs/mcp-and-client-adapters.md @@ -0,0 +1,229 @@ +# 统一 Memory MCP 与客户端接入 + +本页描述 DNA Memory 当前的跨客户端用法、隐私边界和运维方式。所有路径均为 +通用示例；真实 profile、vault、数据库和会话目录必须留在仓库外。 + +## 数据边界 + +- `memory_remember` 执行类型、敏感内容和容量检查后，原子写入受管 Markdown。 +- `memory_reindex` 从 Markdown 重建 SQLite；Markdown 删除后重建会同步删除索引。 +- 原生会话导入器默认只保存会话 ID、项目路径、来源指针、偏移、哈希和计数。 +- 完整 transcript、base64、工具大输出、reasoning 和凭证不会复制到候选队列。 +- 只有安全的 `memory_proposal` 才允许在 daily 维护中结晶为长期记忆。 +- 召回遥测保存查询 SHA-256、客户端、会话 ID、结果数和时间，不保存查询正文。 + +## MCP 工具 + +| 工具 | 说明 | +|---|---| +| `memory_recall(query, limit, client, session_id)` | 召回 active 记忆 | +| `memory_get(memory_id)` | 获取单条记忆及替代关系 | +| `memory_remember(...)` | 写入验证后的结论 | +| `memory_feedback(memory_id, outcome, ...)` | 记录 useful/misleading | +| `memory_close_session(...)` | 记录有界来源指针 | +| `memory_status()` | 查看真源和索引 | +| `memory_reindex()` | 从 Markdown 重建索引 | + +`memory_recall` 支持多词匹配，并结合命中词数、反馈、置信度、重要性和更新时间 +排序。默认最多 20 条；客户端行为 Skill 应进一步限制上下文注入量。 + +### 替代过时结论 + +```json +{ + "type": "project_state", + "summary": "部署已经通过真实跨端召回验收。", + "supersedes": ["mem_old_waiting", "mem_old_unverified"] +} +``` + +所有旧 ID 必须是受管目录内的 active Markdown 记忆。成功后新记忆为 active， +旧记忆标记为 superseded；默认召回不再返回旧结论，`memory_get` 仍可读取历史。 +不要按项目或类型自动推断替代关系。 + +## 独立运行时 + +```bash +git clone https://github.com/AIPMAndy/dna-memory.git \ + "$HOME/.local/share/dna-memory/app" +cd "$HOME/.local/share/dna-memory/app" + +python3 -m venv "$HOME/.local/share/dna-memory/mcp-venv" +"$HOME/.local/share/dna-memory/mcp-venv/bin/pip" install \ + -r requirements-mcp.txt + +mkdir -p "$HOME/.config/dna-memory" +cp docs/profiles/profile.example.json \ + "$HOME/.config/dna-memory/profile.json" +``` + +建议使用独立 venv，避免改变系统 Python 或客户端依赖。 + +## Codex + +```bash +codex mcp add dna-memory \ + --env "DNA_MEMORY_PROFILE=$HOME/.config/dna-memory/profile.json" \ + -- "$HOME/.local/share/dna-memory/mcp-venv/bin/python" \ + "$HOME/.local/share/dna-memory/app/scripts/memory_mcp.py" + +codex mcp get dna-memory +``` + +Codex 没有统一可靠的会话结束 hook，因此可周期运行增量导入器： + +```bash +python3 "$HOME/.local/share/dna-memory/app/scripts/import_codex_rollouts.py" +``` + +它按文件 inode、字节偏移和首行指纹建立检查点，只保存 rollout 级来源指针。 +有限尾部窗口只接受 assistant 输出中的显式提案，不复制逐条消息。 + +## Claude Code + +```bash +claude mcp add --scope user dna-memory \ + -e "DNA_MEMORY_PROFILE=$HOME/.config/dna-memory/profile.json" \ + -- "$HOME/.local/share/dna-memory/mcp-venv/bin/python" \ + "$HOME/.local/share/dna-memory/app/scripts/memory_mcp.py" + +claude mcp get dna-memory +``` + +可把以下非阻塞脚本追加到现有 `SessionStart`、`Stop` 和 `SessionEnd` hooks， +不要覆盖用户已有 hooks： + +```bash +python3 "$HOME/.local/share/dna-memory/app/scripts/client_event_hook.py" +``` + +`Stop` 优先读取官方 `last_assistant_message`，旧版本才回退到 transcript 尾部 +64KB。普通生命周期事件只保存指针。 + +## Claude Desktop + +Claude Desktop 与 Claude Code 的 MCP 配置相互独立。`mcpServers` 必须使用绝对 +路径，以下占位符需要替换： + +```json +{ + "mcpServers": { + "dna-memory": { + "command": "/ABSOLUTE/PATH/mcp-venv/bin/python", + "args": ["/ABSOLUTE/PATH/dna-memory/scripts/memory_mcp.py"], + "env": { + "DNA_MEMORY_PROFILE": "/ABSOLUTE/PATH/profile.json" + } + } + } +} +``` + +已有配置应先 dry-run 迁移： + +```bash +python3 scripts/configure_claude_desktop.py \ + --config "$CLAUDE_DESKTOP_CONFIG" \ + --python "$DNA_MEMORY_PYTHON" \ + --server "$DNA_MEMORY_ROOT/scripts/memory_mcp.py" \ + --profile "$DNA_MEMORY_PROFILE" \ + --backup-dir "$HOME/.local/share/dna-memory/migration-backups" +``` + +确认 `would_add` 或 `would_replace` 后追加 `--apply`。配置器会原子备份与替换， +不会删除旧数据；遇到未知自定义 `dna-memory` 服务时会停止。回滚： + +```bash +python3 scripts/configure_claude_desktop.py \ + --config "$CLAUDE_DESKTOP_CONFIG" \ + --rollback "$CONFIG_BACKUP" +``` + +普通云端聊天没有已验证的稳定本地 transcript 时，覆盖报告会标记 +`explicit-mcp-writeback`。不能把“配置了 MCP”描述为“自动捕获了所有云端聊天”。 + +## Hermes + +```bash +hermes mcp add dna-memory \ + --command "$HOME/.local/share/dna-memory/mcp-venv/bin/python" \ + --env "DNA_MEMORY_PROFILE=$HOME/.config/dna-memory/profile.json" \ + --args "$HOME/.local/share/dna-memory/app/scripts/memory_mcp.py" + +hermes mcp list +hermes mcp test dna-memory +``` + +命名 profile 需要在对应 profile 中重复配置，不能只检查默认 profile。 +`import_hermes_sessions.py` 通过 SQLite 只读 URI 读取 Hermes 状态库，只保存 session +元数据、消息数量和最大 message ID。为了发现显式提案，每个会话只检查最近 8 条 +assistant 消息，不保存未命中标记的正文、reasoning 或工具参数。 + +## 原生历史自动提炼 + +```bash +python3 scripts/import_claudian_sessions.py +python3 scripts/import_claude_desktop_sessions.py +python3 scripts/import_hermes_sessions.py +python3 scripts/import_native_history.py +``` + +普通历史扫描使用客户端白名单、尾部 64KB、最多 12 条消息和每端文件预算。 +`request_dump`、工具结果、配置文件和账号数据不作为聊天正文。检查点独立且幂等； +重复运行时未变化来源应为 `enqueued=0`。 + +来源指针由 `max_candidate_events` 限制，默认 10,000 条待处理普通事件。达到上限 +时拒绝新增普通指针，但不阻塞客户端主任务。不要用扩大上限代替维护。 + +## 维护与价值 + +```bash +python3 dna.py memory status --json +python3 dna.py memory coverage --json +python3 dna.py memory value --json +python3 dna.py memory maintain daily --json +python3 dna.py memory maintain weekly --json +python3 dna.py memory maintain monthly --json +``` + +- daily：结晶安全提案，压缩旧指针，清理过期终态事件。 +- weekly：daily + SQLite 在线备份、有限轮换和 `VACUUM`。 +- monthly：weekly + `PRAGMA integrity_check` 和 Markdown 重建核对。 + +自动化可使用 macOS LaunchAgent、cron 或其他调度器。建议使用通用 label： + +```text +io.dna-memory.native-history-import +io.dna-memory.hermes-import +io.dna-memory.daily +io.dna-memory.weekly +io.dna-memory.monthly +``` + +日志只保留紧凑计数、备份路径和完整性结果，不输出摘要或正文。 + +## Skill 管理 + +```bash +python3 dna.py skills inventory --json +python3 dna.py skills doctor --json +python3 dna.py skills sync --json +python3 dna.py skills sync --apply --json +``` + +`skill_root` 是共享 Skill 真源，注册表决定可分发范围。注册表外的客户端专属 +Skill 永不自动删除或覆盖。详见 [skill-management.md](skill-management.md)。 + +## 验收与回滚 + +部署后至少完成： + +1. `memory_status` 成功。 +2. 客户端实际列出 7 个 MCP 工具。 +3. 一端 `remember` 后同端 `recall` 命中同一 ID。 +4. 另一端能召回该 ID。 +5. `memory_feedback(useful)` 成功。 +6. SQLite `integrity_check` 为 `ok`。 + +回滚时移除客户端 MCP 配置和新增 hook。不要删除 Markdown 真源或共享 Skill +真源；SQLite 可从 Markdown 重建。禁用导入调度不会删除原生会话。 diff --git a/docs/profiles/profile.example.json b/docs/profiles/profile.example.json new file mode 100644 index 0000000..cfb0270 --- /dev/null +++ b/docs/profiles/profile.example.json @@ -0,0 +1,21 @@ +{ + "knowledge_root": "~/Documents/DNA-Memory-Vault", + "database_path": "~/.local/share/dna-memory/memory.db", + "managed_memory_dir": "Memory", + "skill_root": "~/.agents/skills", + "skill_registry": "~/.config/dna-memory/skills.json", + "platform_skill_roots": { + "codex": "~/.codex/skills", + "claude": "~/.claude/skills", + "hermes": "~/.hermes/skills" + }, + "warning_bytes": 104857600, + "hard_bytes": 262144000, + "max_records": 10000, + "max_candidate_events": 10000, + "backup_dir": "~/.local/share/dna-memory/backups/managed", + "backup_keep": 8, + "claudian_session_dirs": [], + "claude_desktop_session_dirs": [], + "hermes_state_db": null +} diff --git a/docs/skill-management.md b/docs/skill-management.md new file mode 100644 index 0000000..77d34c8 --- /dev/null +++ b/docs/skill-management.md @@ -0,0 +1,65 @@ +# 共享 Skill 管理 + +DNA Memory 将“长期记忆”和“行为 Skill”分开治理： + +- `knowledge_root/managed_memory_dir` 是验证后结论的 Markdown 真源。 +- `skill_root` 是共享 Skill 真源，推荐 `~/.agents/skills`。 +- `platform_skill_roots` 是 Codex、Claude Code、Hermes 的加载目录。 +- `skill_registry` 只声明 DNA Memory 有权分发的共享 Skill。 +- 注册表外的目录属于平台专属或未管理内容，不会被删除或覆盖。 + +## 注册表 + +```json +{ + "skills": { + "dna-memory-loop": { + "targets": ["codex", "claude", "hermes"] + } + } +} +``` + +推荐将仓库内的 `skills/dna-memory-loop` 放入共享真源。它统一三端行为： + +1. 依赖历史的任务开始前，以独立关键词召回。 +2. 只有真正使用的召回结果才提交反馈。 +3. 只有验证后的长期结论才写回。 +4. 记忆服务故障不阻塞主任务。 +5. 不把完整 transcript、凭证或大型工具输出写入记忆。 + +## 命令 + +```bash +python3 dna.py skills inventory --json +python3 dna.py skills doctor --json +python3 dna.py skills sync --json +python3 dna.py skills sync --apply --json +``` + +`sync` 默认 dry-run。`--apply` 只为缺失目标创建符号链接，不会覆盖现有目录。 + +| 状态 | 含义 | 自动操作 | +|---|---|---| +| `shared` | 正确指向共享真源 | 无 | +| `platform` | 注册表外的平台专属 Skill | 保留 | +| `shadowed` | 内容相同但不是链接 | 仅报告 | +| `conflict` | 同名内容不同 | 阻断 | +| `broken_link` | 链接目标不存在 | 仅报告 | + +## 候选提案 + +客户端不能直接调用 `memory_remember` 时，可在最终消息中输出： + +```text +DNA_MEMORY_PROPOSAL {"type":"decision","summary":"verified reusable conclusion","confidence":"high","importance":0.8} +``` + +每个会话最多 3 条，每条不超过 800 字符。提案不是长期记忆；daily 维护仍会 +执行类型、敏感信息、容量和去重检查。普通 Claude Desktop 云端聊天没有稳定 +本地正文来源时，应使用显式 MCP 写回，不能假设后台自动捕获。 + +## 回滚 + +只删除客户端目录中的受管符号链接，不要删除 `skill_root` 真源。运行 +`skills doctor` 确认没有断链或冲突。未知 Skill 始终保留。 diff --git a/memory/README.md b/memory/README.md new file mode 100644 index 0000000..d944a36 --- /dev/null +++ b/memory/README.md @@ -0,0 +1,8 @@ +# Runtime data + +This directory is intentionally empty in source control. DNA Memory databases, +events, genes, capsules, and other user-generated memory data are local runtime +artifacts and must never be committed. + +The recommended database location is outside the repository: +`~/.local/share/dna-memory/memory.db`. diff --git a/memory/capsules.json b/memory/capsules.json deleted file mode 100644 index 0637a08..0000000 --- a/memory/capsules.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/memory/events.jsonl b/memory/events.jsonl deleted file mode 100644 index 4f15069..0000000 --- a/memory/events.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"timestamp": 1776441396.340371, "type": "gene_created", "details": {"gene_id": "gene_1776441396340", "name": "飞书 API 限流处理"}} diff --git a/memory/genes.json b/memory/genes.json deleted file mode 100644 index 0025707..0000000 --- a/memory/genes.json +++ /dev/null @@ -1,37 +0,0 @@ -[ - { - "id": "gene_1776441396340", - "name": "飞书 API 限流处理", - "description": "检测并处理飞书 API 限流", - "triggers": [ - { - "type": "error", - "keywords": [ - "飞书", - "限流", - "429" - ] - } - ], - "actions": [ - { - "type": "remember", - "content": "分段请求，每批 50 条" - }, - { - "type": "remember", - "content": "添加 1 秒延迟" - } - ], - "tags": [ - "feishu", - "api", - "rate-limit" - ], - "created": 1776441396.340255, - "updated": 1776441396.34049, - "usage_count": 1, - "success_count": 1, - "failure_count": 0 - } -] \ No newline at end of file diff --git a/memory/meta_memory.json b/memory/meta_memory.json deleted file mode 100644 index 0b173a5..0000000 --- a/memory/meta_memory.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "created_at": 1783039525.94028, - "total_memories": 0, - "quality_trend": [], - "recall_accuracy": 0.0, - "false_positive_rate": 0.0, - "distillation_count": 0, - "reinforcement_cycles": 0, - "evolution_milestones": [], - "health_checks": [ - { - "timestamp": 1783039525.9420328, - "status": "healthy", - "current_quality": 0.6400313798515953, - "total_memories": 10, - "low_quality_count": 0, - "issues": [] - } - ] -} \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..75b4ea8 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1 @@ +pytest>=8,<9 diff --git a/requirements-mcp.txt b/requirements-mcp.txt new file mode 100644 index 0000000..4a3501c --- /dev/null +++ b/requirements-mcp.txt @@ -0,0 +1,2 @@ +mcp>=1.0,<2 +PyYAML>=6,<7 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1910948 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +PyYAML>=6.0,<7 diff --git a/scripts/bounded_proposals.py b/scripts/bounded_proposals.py new file mode 100644 index 0000000..a0a95cd --- /dev/null +++ b/scripts/bounded_proposals.py @@ -0,0 +1,65 @@ +"""Extract explicit, bounded memory proposals without retaining transcripts.""" + +import json +import re +from pathlib import Path + +from scripts.markdown_memory import SUPPORTED_TYPES +from scripts.policy import inspect_content + + +MARKER = re.compile(r"DNA_MEMORY_PROPOSAL\s*(\{.*?\})", re.DOTALL) +MAX_SUMMARY_CHARS = 800 +DEFAULT_MAX_PROPOSALS = 3 +DEFAULT_TAIL_BYTES = 64 * 1024 + + +def extract_proposals(text, max_proposals=DEFAULT_MAX_PROPOSALS): + """Return only valid JSON proposals explicitly emitted by an agent.""" + found = [] + for match in MARKER.finditer(str(text or "")): + if len(found) >= max_proposals: + break + try: + proposal = json.loads(match.group(1)) + except (TypeError, json.JSONDecodeError): + continue + if not isinstance(proposal, dict): + continue + summary = str(proposal.get("summary", "")).strip() + if proposal.get("type") not in SUPPORTED_TYPES: + continue + if not summary or len(summary) > MAX_SUMMARY_CHARS: + continue + if not inspect_content(summary).allowed: + continue + confidence = proposal.get("confidence") + if confidence is not None and confidence not in {"high", "medium", "low"}: + continue + importance = proposal.get("importance") + if importance is not None and ( + isinstance(importance, bool) + or not isinstance(importance, (int, float)) + or not 0.0 <= float(importance) <= 1.0): + continue + item = {"type": proposal["type"], "summary": summary} + for key in ("confidence", "importance"): + if key in proposal: + item[key] = proposal[key] + found.append(item) + return found + + +def read_tail_proposals(path, max_bytes=DEFAULT_TAIL_BYTES, + max_proposals=DEFAULT_MAX_PROPOSALS): + """Read at most the tail window; never return source text.""" + path = Path(path).expanduser() + try: + with path.open("rb") as handle: + handle.seek(0, 2) + size = handle.tell() + handle.seek(max(0, size - max_bytes)) + data = handle.read(max_bytes) + except OSError: + return [] + return extract_proposals(data.decode("utf-8", errors="ignore"), max_proposals) diff --git a/scripts/candidate_events.py b/scripts/candidate_events.py new file mode 100644 index 0000000..ead2937 --- /dev/null +++ b/scripts/candidate_events.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Bounded, idempotent queue of client event pointers.""" + +import sqlite3 +from pathlib import Path + +from scripts.policy import inspect_content + + +class CandidateEventQueue: + def __init__(self, path, max_events=10000): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + self.max_events = int(max_events) + self.connection = sqlite3.connect(str(path)) + self.connection.execute(""" + CREATE TABLE IF NOT EXISTS candidate_events ( + event_id TEXT PRIMARY KEY, client TEXT NOT NULL, + event_type TEXT NOT NULL, session_id TEXT, + project_path TEXT, source_ref TEXT, source_hash TEXT, + excerpt TEXT, status TEXT NOT NULL DEFAULT 'pending', + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + """) + columns = { + row[1] for row in self.connection.execute( + "PRAGMA table_info(candidate_events)" + ).fetchall() + } + additions = { + "memory_type": "TEXT", "confidence": "TEXT", + "importance": "REAL", "processed_at": "TEXT", + "memory_id": "TEXT", "error": "TEXT", + } + for name, definition in additions.items(): + if name not in columns: + self.connection.execute( + "ALTER TABLE candidate_events ADD COLUMN {} {}".format(name, definition) + ) + self.connection.execute(""" + CREATE TABLE IF NOT EXISTS import_checkpoints ( + source_ref TEXT PRIMARY KEY, inode INTEGER, offset INTEGER NOT NULL DEFAULT 0, + source_hash TEXT, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + """) + self.connection.commit() + + def enqueue(self, event): + try: + self.connection.execute("BEGIN IMMEDIATE") + if event.get("event_type") != "memory_proposal": + pending = self.connection.execute( + "SELECT COUNT(*) FROM candidate_events WHERE status='pending' " + "AND event_type!='memory_proposal'" + ).fetchone()[0] + if pending >= self.max_events: + self.connection.rollback() + return False + excerpt = event.get("excerpt") + if excerpt and not inspect_content(str(excerpt)).allowed: + excerpt = None + cursor = self.connection.execute(""" + INSERT OR IGNORE INTO candidate_events + (event_id, client, event_type, session_id, project_path, source_ref, + source_hash, excerpt, memory_type, confidence, importance) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, (event["event_id"], event["client"], event["event_type"], + event.get("session_id"), event.get("project_path"), event.get("source_ref"), + event.get("source_hash"), excerpt, event.get("memory_type"), + event.get("confidence"), event.get("importance"))) + self.connection.commit() + return cursor.rowcount == 1 + except Exception: + self.connection.rollback() + raise + + def get_checkpoint(self, source_ref): + return self.connection.execute( + "SELECT inode, offset, source_hash FROM import_checkpoints WHERE source_ref=?", + (str(source_ref),), + ).fetchone() + + def update_checkpoint(self, source_ref, inode, offset, source_hash): + self.connection.execute(""" + INSERT INTO import_checkpoints (source_ref, inode, offset, source_hash) + VALUES (?, ?, ?, ?) + ON CONFLICT(source_ref) DO UPDATE SET + inode=excluded.inode, offset=excluded.offset, + source_hash=excluded.source_hash, updated_at=CURRENT_TIMESTAMP + """, (str(source_ref), int(inode), int(offset), source_hash)) + self.connection.commit() diff --git a/scripts/check_public_safety.py b/scripts/check_public_safety.py new file mode 100644 index 0000000..01b03d9 --- /dev/null +++ b/scripts/check_public_safety.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Fail when the public source tree contains private deployment data.""" + +import argparse +import re +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SKIP_DIRS = {".git", ".pytest_cache", "__pycache__", ".venv", "venv"} +SKIP_FILES = {Path(__file__).resolve()} + +FORBIDDEN_LITERALS = ( + "/" + "Users/", + "Andy" + "DATA", + "com." + "andy", + "andy" + "-profile", + "andy" + "-memory-loop", + "andy" + "-workflow", + "andy" + "-zsxq", + "andy" + "-mac", + "-Users-" + "andy-", +) + +SECRET_PATTERNS = ( + ("private key", re.compile("-----BEGIN " + r"(?:RSA |EC |OPENSSH )?" + "PRIVATE KEY-----")), + ("OpenAI-style token", re.compile(r"\bsk-[A-Za-z0-9_-]{20,}\b")), + ("GitHub token", re.compile(r"\bgh[pousr]_[A-Za-z0-9_]{20,}\b")), + ("Slack token", re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b")), + ("AWS access key", re.compile(r"\bAKIA[0-9A-Z]{16}\b")), + ( + "assigned secret", + re.compile( + r"(?i)\b(?:api[_-]?key|access[_-]?token|secret[_-]?key|password)" + r"\s*[:=]\s*['\"]?[A-Za-z0-9_./+=-]{12,}" + ), + ), +) + + +def source_files(root: Path): + for path in root.rglob("*"): + if not path.is_file() or path.resolve() in SKIP_FILES: + continue + if any(part in SKIP_DIRS for part in path.relative_to(root).parts): + continue + yield path + + +def inspect_tree(root: Path): + findings = [] + for path in source_files(root): + try: + raw = path.read_bytes() + if b"\0" in raw: + continue + text = raw.decode("utf-8") + except (OSError, UnicodeDecodeError): + continue + relative = path.relative_to(root) + for literal in FORBIDDEN_LITERALS: + if literal.lower() in text.lower() or literal.lower() in str(relative).lower(): + findings.append((str(relative), "private identifier", literal)) + for name, pattern in SECRET_PATTERNS: + match = pattern.search(text) + if match: + findings.append((str(relative), name, "content redacted")) + return findings + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=ROOT) + args = parser.parse_args(argv) + findings = inspect_tree(args.root.resolve()) + if findings: + for path, kind, detail in findings: + print("{}: {} ({})".format(path, kind, detail), file=sys.stderr) + print("public safety check failed: {} finding(s)".format(len(findings)), file=sys.stderr) + return 1 + print("public safety check passed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/client_coverage.py b/scripts/client_coverage.py new file mode 100644 index 0000000..5545c84 --- /dev/null +++ b/scripts/client_coverage.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +"""Report evidence for each supported client surface without reading transcripts.""" + +import json +from pathlib import Path +import sqlite3 +import subprocess +import time +from urllib.parse import quote + +from scripts.import_native_history import configured_paths, source_files + + +NATIVE_SURFACES = { + "codex-desktop": "codex", + "codex-cli": "codex", + "claude-code-desktop": "claude-code", + "claude-code-cli": "claude-code", + "claude-cowork": "claude-desktop", + "hermes-desktop": "hermes", + "hermes-cli": "hermes", + "hermes-gateway": "hermes", +} + +JSONL_METADATA_BYTES = 512 * 1024 +JSONL_LINE_BYTES = 256 * 1024 +HERMES_LOCAL_SOURCES = {"desktop", "cli", "tui", "subagent"} + + +def _query_one(connection, sql, parameters=()): + if connection is None: + return None + try: + return connection.execute(sql, parameters).fetchone() + except sqlite3.Error: + return None + + +def _checkpoint_status(connection, client, spec, paths, min_age_seconds=120): + now = time.time() + eligible = [] + checkpointed = 0 + eligible_checkpointed = 0 + for path in paths: + stat = path.stat() + checkpoint = _query_one( + connection, + "SELECT inode, offset, source_hash FROM import_checkpoints WHERE source_ref=?", + ("native-auto:{}:{}".format(client, path.expanduser().resolve()),), + ) + current = bool( + checkpoint and checkpoint[0] == stat.st_ino and checkpoint[1] == stat.st_size + ) + if current: + checkpointed += 1 + if now - stat.st_mtime < int(min_age_seconds): + continue + eligible.append(path) + if current: + eligible_checkpointed += 1 + row = _query_one( + connection, + "SELECT MAX(updated_at) FROM import_checkpoints WHERE source_ref LIKE ?", + ("native-auto:{}:%".format(client),), + ) + roots_exist = any(Path(root).expanduser().exists() for root in spec.roots) + return { + "source_exists": roots_exist, + "source_files": len(paths), + "eligible_files": len(eligible), + "checkpointed_files": checkpointed, + "eligible_checkpointed_files": eligible_checkpointed, + "checkpoint_complete": roots_exist and eligible_checkpointed == len(eligible), + "last_import_at": row[0] if row else None, + } + + +def _jsonl_marker(path, client): + consumed = 0 + try: + with path.open("rb") as handle: + while consumed < JSONL_METADATA_BYTES: + line = handle.readline(min(JSONL_LINE_BYTES, JSONL_METADATA_BYTES - consumed)) + if not line: + break + consumed += len(line) + if not line.endswith(b"\n") and len(line) == JSONL_LINE_BYTES: + break + try: + item = json.loads(line) + except (json.JSONDecodeError, UnicodeDecodeError): + continue + if client == "codex" and item.get("type") == "session_meta": + payload = item.get("payload") or {} + originator = str(payload.get("originator") or "") + raw_source = payload.get("source") + if isinstance(raw_source, str): + source = raw_source + elif isinstance(raw_source, dict) and "subagent" in raw_source: + source = "subagent" + elif raw_source: + source = "structured" + else: + source = "" + if originator == "Codex Desktop": + return "codex-desktop", "{}/{}".format(originator, source or "unknown") + if originator == "codex-tui" or source == "cli": + return "codex-cli", "{}/{}".format(originator or "unknown", source) + return None, "{}/{}".format(originator or "unknown", source or "unknown") + if client == "claude-code": + entrypoint = item.get("entrypoint") + if entrypoint == "claude-desktop-3p": + return "claude-code-desktop", entrypoint + if entrypoint == "sdk-cli": + return "claude-code-cli", entrypoint + if entrypoint: + return None, str(entrypoint) + except OSError: + pass + return None, None + + +def _native_entry_evidence(files): + surface_paths = { + "codex-desktop": [], + "codex-cli": [], + "claude-code-desktop": [], + "claude-code-cli": [], + } + markers = {surface: {} for surface in surface_paths} + unclassified = {"codex": 0, "claude-code": 0} + for client in ("codex", "claude-code"): + for path in files.get(client, []): + surface, marker = _jsonl_marker(path, client) + if surface is None: + unclassified[client] += 1 + continue + surface_paths[surface].append(path) + markers[surface][marker] = markers[surface].get(marker, 0) + 1 + return surface_paths, markers, unclassified + + +def _contains_mcp(paths): + for path in paths: + try: + text = path.read_text(encoding="utf-8", errors="ignore") + except OSError: + continue + if "dna-memory" in text and "memory_mcp.py" in text: + return True + return False + + +def _mcp_status(home): + home = Path(home) + application_support = home / "Library" / "Application Support" + return { + "codex": _contains_mcp((home / ".codex" / "config.toml",)), + "claude-code": _contains_mcp(( + home / ".claude" / "mcp.json", + home / ".claude" / "settings.json", + home / ".claude.json", + )), + "claude-desktop": _contains_mcp(tuple( + application_support.glob("Claude*/claude_desktop_config.json") + )), + "hermes": _contains_mcp((home / ".hermes" / "config.yaml",)), + } + + +def _launch_agent(home, label): + plist = Path(home) / "Library" / "LaunchAgents" / (label + ".plist") + loaded = False + last_exit_code = None + if plist.is_file(): + try: + result = subprocess.run( + ["launchctl", "print", "gui/{}/{}".format(Path(home).stat().st_uid, label)], + capture_output=True, text=True, check=False, + ) + except FileNotFoundError: + return {"installed": True, "loaded": False, "last_exit_code": None} + loaded = result.returncode == 0 + for line in result.stdout.splitlines(): + if "last exit code =" in line: + try: + last_exit_code = int(line.rsplit("=", 1)[1].strip()) + except ValueError: + pass + break + return {"installed": plist.is_file(), "loaded": loaded, "last_exit_code": last_exit_code} + + +def _launch_agent_by_suffix(home, suffix, fallback_label): + directory = Path(home) / "Library" / "LaunchAgents" + candidates = sorted(directory.glob("*{}.plist".format(suffix))) + label = candidates[0].stem if candidates else fallback_label + return _launch_agent(home, label) + + +def _hermes_state_status(config, index_connection): + path = config.hermes_state_db + if not path or not path.is_file(): + return { + "exists": False, "sessions": 0, "checkpointed_sessions": 0, + "source_counts": {}, + } + uri = "file:{}?mode=ro".format(path.resolve()) + connection = sqlite3.connect(uri, uri=True) + try: + source_counts = dict(connection.execute( + "SELECT source, COUNT(*) FROM sessions GROUP BY source" + ).fetchall()) + sessions = sum(source_counts.values()) + finally: + connection.close() + row = _query_one( + index_connection, + "SELECT COUNT(*) FROM import_checkpoints WHERE source_ref LIKE ?", + ("hermes:{}#%".format(path.resolve()),), + ) + checkpointed = row[0] if row else 0 + return { + "exists": True, + "sessions": sessions, + "checkpointed_sessions": checkpointed, + "checkpoint_complete": checkpointed >= sessions, + "source_counts": source_counts, + } + + +def _file_entry_evidence(paths, markers): + return { + "verified": bool(paths), + "matched_files": len(paths), + "markers": markers, + } + + +def _hermes_entry_evidence(source_counts, accepted=None, gateway=False): + if gateway: + markers = { + source: count for source, count in source_counts.items() + if source not in HERMES_LOCAL_SOURCES + } + else: + markers = { + source: source_counts[source] for source in accepted or () + if source_counts.get(source) + } + matched = sum(markers.values()) + return {"verified": matched > 0, "matched_sessions": matched, "markers": markers} + + +def build_coverage_report(config, home=None, paths_by_client=None, min_age_seconds=120): + home = Path(home or Path.home()) + specs = paths_by_client or configured_paths(config) + files = source_files(specs) + index_connection = None + if config.database_path.is_file(): + uri = "file:{}?mode=ro".format(quote(str(config.database_path.resolve()), safe="/")) + index_connection = sqlite3.connect(uri, uri=True) + try: + native = { + client: _checkpoint_status( + index_connection, client, specs[client], paths, min_age_seconds + ) + for client, paths in files.items() + } + surface_paths, surface_markers, unclassified = _native_entry_evidence(files) + surface_sources = { + surface: _checkpoint_status( + index_connection, NATIVE_SURFACES[surface], + specs[NATIVE_SURFACES[surface]], paths, min_age_seconds, + ) + for surface, paths in surface_paths.items() + } + for client, count in unclassified.items(): + if client in native: + native[client]["unclassified_files"] = count + hermes_state = _hermes_state_status(config, index_connection) + finally: + if index_connection is not None: + index_connection.close() + mcp = _mcp_status(home) + native_agent = _launch_agent_by_suffix( + home, ".dna-memory-native-history-import", + "io.dna-memory.native-history-import", + ) + hermes_agent = _launch_agent_by_suffix( + home, ".dna-memory-hermes-import", "io.dna-memory.hermes-import" + ) + surfaces = {} + for surface, client in NATIVE_SURFACES.items(): + source = native.get(client, { + "source_exists": False, + "source_files": 0, "eligible_files": 0, "checkpointed_files": 0, + "checkpoint_complete": False, "last_import_at": None, + }) + surfaces[surface] = { + "capture_mode": "automatic-structured" if client == "hermes" else "automatic-native", + "automatic_capture": True, + "source": source, + "structured_source": hermes_state if client == "hermes" else None, + "mcp_configured": mcp[client], + "launch_agent": hermes_agent if client == "hermes" else native_agent, + } + for surface in ("codex-desktop", "codex-cli", "claude-code-desktop", "claude-code-cli"): + client = NATIVE_SURFACES[surface] + paths = surface_paths[surface] + surfaces[surface]["source"] = surface_sources[surface] + surfaces[surface]["entry_evidence"] = _file_entry_evidence( + paths, surface_markers[surface] + ) + source_counts = hermes_state.get("source_counts", {}) + surfaces["hermes-desktop"]["entry_evidence"] = _hermes_entry_evidence( + source_counts, accepted=("desktop",) + ) + surfaces["hermes-cli"]["entry_evidence"] = _hermes_entry_evidence( + source_counts, accepted=("cli", "tui") + ) + surfaces["hermes-gateway"]["entry_evidence"] = _hermes_entry_evidence( + source_counts, gateway=True + ) + surfaces["claude-desktop-cloud"] = { + "capture_mode": "explicit-mcp-writeback", + "automatic_capture": False, + "reason": "no stable local transcript source confirmed", + "source": None, + "structured_source": None, + "mcp_configured": mcp["claude-desktop"], + "launch_agent": None, + } + return {"surfaces": surfaces, "native_sources": native, "hermes_state": hermes_state} + + +def main(): + from scripts.config import load_config + + print(json.dumps(build_coverage_report(load_config()), ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/client_event_hook.py b/scripts/client_event_hook.py new file mode 100644 index 0000000..089e476 --- /dev/null +++ b/scripts/client_event_hook.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Nonblocking Claude lifecycle hook that stores metadata pointers only.""" + +import hashlib +import json +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from scripts.candidate_events import CandidateEventQueue +from scripts.bounded_proposals import extract_proposals, read_tail_proposals +from scripts.config import load_config + + +def capture_payload(payload, database_path): + session_id = str(payload.get("session_id", "")).strip() + event_type = str(payload.get("hook_event_name", "")).strip() + if not session_id or not event_type: + return False + source_ref = payload.get("transcript_path") + source_size = -1 + if source_ref: + try: + source_size = Path(source_ref).stat().st_size + except OSError: + pass + identity = "|".join((session_id, event_type, str(source_ref or ""), str(source_size))) + digest = hashlib.sha256(identity.encode()).hexdigest() + queue = CandidateEventQueue(database_path) + try: + enqueued = queue.enqueue({ + "event_id": "claude_{}".format(digest[:24]), + "client": "claude", + "event_type": event_type, + "session_id": session_id, + "project_path": payload.get("cwd"), + "source_ref": source_ref, + "source_hash": digest, + }) + assistant_message = payload.get("last_assistant_message") + if isinstance(assistant_message, str): + proposals = extract_proposals(assistant_message) + proposal_ref = source_ref or "hook:last_assistant_message" + else: + proposals = read_tail_proposals(source_ref) if source_ref else [] + proposal_ref = source_ref + for index, proposal in enumerate(proposals): + proposal_hash = hashlib.sha256( + (digest + str(index) + proposal["summary"]).encode() + ).hexdigest() + enqueued = queue.enqueue({ + "event_id": "claude_proposal_{}".format(proposal_hash[:24]), + "client": "claude", + "event_type": "memory_proposal", + "session_id": session_id, + "project_path": payload.get("cwd"), + "source_ref": "{}#proposal={}".format(proposal_ref, index), + "source_hash": proposal_hash, + "excerpt": proposal["summary"], + "memory_type": proposal["type"], + "confidence": proposal.get("confidence"), + "importance": proposal.get("importance"), + }) or enqueued + return enqueued + finally: + queue.connection.close() + + +def main(): + try: + payload = json.load(sys.stdin) + capture_payload(payload, load_config().database_path) + except Exception: + pass + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/config.py b/scripts/config.py new file mode 100644 index 0000000..fd6e92f --- /dev/null +++ b/scripts/config.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Portable configuration for the unified memory and skill tools.""" + +import json +import os +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, Optional + + +DEFAULTS = { + "knowledge_root": "~/Documents/DNA-Memory-Vault", + "database_path": "~/.local/share/dna-memory/memory.db", + "managed_memory_dir": "Memory", + "skill_root": "~/.agents/skills", + "skill_registry": "~/.config/dna-memory/skills.json", + "platform_skill_roots": {}, + "warning_bytes": 100 * 1024 * 1024, + "hard_bytes": 250 * 1024 * 1024, + "max_records": 10000, + "max_candidate_events": 10000, + "backup_dir": "~/.local/share/dna-memory/backups/managed", + "backup_keep": 8, + "claudian_session_dirs": [], + "claude_desktop_session_dirs": [], + "hermes_state_db": None, +} +DEFAULT_PROFILE = Path.home() / ".config" / "dna-memory" / "profile.json" + + +@dataclass(frozen=True) +class DNAConfig: + knowledge_root: Path + database_path: Path + managed_memory_dir: str + skill_root: Path + skill_registry: Path + platform_skill_roots: Dict[str, Path] = field(default_factory=dict) + warning_bytes: int = DEFAULTS["warning_bytes"] + hard_bytes: int = DEFAULTS["hard_bytes"] + max_records: int = DEFAULTS["max_records"] + max_candidate_events: int = DEFAULTS["max_candidate_events"] + backup_dir: Path = Path(DEFAULTS["backup_dir"]).expanduser() + backup_keep: int = DEFAULTS["backup_keep"] + claudian_session_dirs: tuple = () + claude_desktop_session_dirs: tuple = () + hermes_state_db: Optional[Path] = None + + +def _path(value: str) -> Path: + return Path(os.path.expandvars(value)).expanduser() + + +def load_config(profile_path: Optional[Path] = None) -> DNAConfig: + values = dict(DEFAULTS) + selected = profile_path or os.getenv("DNA_MEMORY_PROFILE") + if not selected and DEFAULT_PROFILE.is_file(): + selected = DEFAULT_PROFILE + if selected: + profile = Path(selected).expanduser() + with profile.open(encoding="utf-8") as handle: + values.update(json.load(handle)) + roots = {name: _path(path) for name, path in values.get("platform_skill_roots", {}).items()} + return DNAConfig( + knowledge_root=_path(values["knowledge_root"]), + database_path=_path(values["database_path"]), + managed_memory_dir=str(values["managed_memory_dir"]), + skill_root=_path(values["skill_root"]), + skill_registry=_path(values["skill_registry"]), + platform_skill_roots=roots, + warning_bytes=int(values["warning_bytes"]), + hard_bytes=int(values["hard_bytes"]), + max_records=int(values["max_records"]), + max_candidate_events=int(values["max_candidate_events"]), + backup_dir=_path(values["backup_dir"]), + backup_keep=int(values["backup_keep"]), + claudian_session_dirs=tuple(_path(path) for path in values.get("claudian_session_dirs", [])), + claude_desktop_session_dirs=tuple( + _path(path) for path in values.get("claude_desktop_session_dirs", []) + ), + hermes_state_db=_path(values["hermes_state_db"]) if values.get("hermes_state_db") else None, + ) diff --git a/scripts/configure_claude_desktop.py b/scripts/configure_claude_desktop.py new file mode 100644 index 0000000..085593f --- /dev/null +++ b/scripts/configure_claude_desktop.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Safely migrate Claude Desktop from generic memory to DNA Memory MCP.""" + +import argparse +from datetime import datetime +import json +import os +from pathlib import Path +import shutil +import sys +import tempfile + + +GENERIC_COMMAND = "mcp-server-memory" + + +def _load_config(path): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError("invalid Claude Desktop config: {}".format(error)) + if not isinstance(payload, dict) or not isinstance(payload.get("mcpServers"), dict): + raise ValueError("Claude Desktop config requires an mcpServers object") + return payload + + +def _atomic_bytes(path, content, mode=None): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + handle = tempfile.NamedTemporaryFile( + prefix=path.name + ".", suffix=".tmp", dir=str(path.parent), delete=False + ) + temp = Path(handle.name) + try: + with handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + if mode is not None: + os.chmod(str(temp), mode) + os.replace(str(temp), str(path)) + finally: + if temp.exists(): + temp.unlink() + + +def configure( + config, python, server, profile, legacy_memory=None, backup_dir=None, + apply=False, replace_custom=False, stamp=None, +): + config = Path(config).expanduser() + payload = _load_config(config) + desired = { + "command": str(Path(python).expanduser()), + "args": [str(Path(server).expanduser())], + "env": {"DNA_MEMORY_PROFILE": str(Path(profile).expanduser())}, + } + current = payload["mcpServers"].get("dna-memory") + if current == desired: + return {"status": "already_configured", "changed": False} + if current is None: + action = "add" + elif isinstance(current, dict) and current.get("command") == GENERIC_COMMAND: + action = "replace" + elif not replace_custom: + raise ValueError("custom dna-memory server requires --replace-custom") + else: + action = "replace" + if not apply: + return {"status": "would_{}".format(action), "changed": False} + + stamp = stamp or datetime.now().strftime("%Y%m%dT%H%M%S") + backup_dir = Path(backup_dir or config.parent / "dna-memory-backups").expanduser() + backup_dir.mkdir(parents=True, exist_ok=True) + config_backup = backup_dir / "{}.{}.bak".format(config.name, stamp) + shutil.copy2(str(config), str(config_backup)) + result = { + "status": "{}ed".format(action) if action == "add" else "replaced", + "changed": True, + "config_backup": str(config_backup), + "legacy_backup": None, + } + if legacy_memory: + legacy_memory = Path(legacy_memory).expanduser() + if legacy_memory.is_file(): + legacy_backup = backup_dir / "{}.{}.bak".format(legacy_memory.name, stamp) + shutil.copy2(str(legacy_memory), str(legacy_backup)) + result["legacy_backup"] = str(legacy_backup) + + payload["mcpServers"]["dna-memory"] = desired + content = (json.dumps(payload, ensure_ascii=False, indent=2) + "\n").encode() + _atomic_bytes(config, content, config.stat().st_mode & 0o777) + return result + + +def rollback(config, backup): + config = Path(config).expanduser() + backup = Path(backup).expanduser() + if not backup.is_file(): + raise ValueError("config backup not found: {}".format(backup)) + mode = config.stat().st_mode & 0o777 if config.exists() else backup.stat().st_mode & 0o777 + _atomic_bytes(config, backup.read_bytes(), mode) + return {"status": "rolled_back", "changed": True, "backup": str(backup)} + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True) + parser.add_argument("--python") + parser.add_argument("--server") + parser.add_argument("--profile") + parser.add_argument("--legacy-memory") + parser.add_argument("--backup-dir") + parser.add_argument("--apply", action="store_true") + parser.add_argument("--replace-custom", action="store_true") + parser.add_argument("--rollback") + args = parser.parse_args(argv) + try: + if args.rollback: + result = rollback(args.config, args.rollback) + else: + if not all((args.python, args.server, args.profile)): + parser.error("--python, --server, and --profile are required") + result = configure( + args.config, args.python, args.server, args.profile, + legacy_memory=args.legacy_memory, backup_dir=args.backup_dir, + apply=args.apply, replace_custom=args.replace_custom, + ) + except ValueError as error: + print(json.dumps({"error": str(error)}, ensure_ascii=False), file=sys.stderr) + return 2 + print(json.dumps(result, ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/import_claude_desktop_sessions.py b/scripts/import_claude_desktop_sessions.py new file mode 100644 index 0000000..2b4380a --- /dev/null +++ b/scripts/import_claude_desktop_sessions.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Capture bounded pointers from Claude Desktop and Cowork session metadata.""" + +import argparse +import hashlib +import json +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from scripts.candidate_events import CandidateEventQueue +from scripts.config import load_config + + +def import_sessions(paths, queue): + result = {"files": 0, "enqueued": 0, "skipped": 0} + for root in paths: + root = Path(root).expanduser() + if not root.is_dir(): + continue + for path in sorted(root.rglob("local_*.json")): + result["files"] += 1 + raw = path.read_bytes() + digest = hashlib.sha256(raw).hexdigest() + source_ref = str(path.resolve()) + checkpoint = queue.get_checkpoint(source_ref) + if checkpoint and checkpoint[2] == digest: + result["skipped"] += 1 + continue + try: + payload = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError): + result["skipped"] += 1 + continue + session_id = str( + payload.get("sessionId") or payload.get("cliSessionId") or "" + ).strip() + if not session_id: + result["skipped"] += 1 + continue + project_path = payload.get("cwd") + if project_path is not None: + project_path = str(project_path) + identity = "{}|{}".format(source_ref, digest) + event_id = "claude_desktop_{}".format( + hashlib.sha256(identity.encode()).hexdigest()[:24] + ) + if queue.enqueue({ + "event_id": event_id, + "client": "claude-desktop", + "event_type": "session_updated", + "session_id": session_id, + "project_path": project_path, + "source_ref": source_ref, + "source_hash": digest, + }): + result["enqueued"] += 1 + stat = path.stat() + queue.update_checkpoint(source_ref, stat.st_ino, stat.st_size, digest) + return result + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("paths", nargs="*") + args = parser.parse_args(argv) + config = load_config() + paths = args.paths or list(config.claude_desktop_session_dirs) + queue = CandidateEventQueue(config.database_path, config.max_candidate_events) + try: + result = import_sessions(paths, queue) + finally: + queue.connection.close() + print(json.dumps(result, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/import_claudian_sessions.py b/scripts/import_claudian_sessions.py new file mode 100644 index 0000000..66c7cf6 --- /dev/null +++ b/scripts/import_claudian_sessions.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Incrementally capture bounded pointers from Claudian session metadata.""" + +import argparse +import hashlib +import json +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from scripts.candidate_events import CandidateEventQueue +from scripts.config import load_config + + +def import_sessions(paths, queue, project_path=None): + result = {"files": 0, "enqueued": 0, "skipped": 0} + for root in paths: + root = Path(root).expanduser() + if not root.is_dir(): + continue + for path in sorted(root.glob("*.meta.json")): + result["files"] += 1 + raw = path.read_bytes() + digest = hashlib.sha256(raw).hexdigest() + checkpoint = queue.get_checkpoint(str(path.resolve())) + if checkpoint and checkpoint[2] == digest: + result["skipped"] += 1 + continue + try: + payload = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError): + result["skipped"] += 1 + continue + session_id = str(payload.get("id") or payload.get("sessionId") or "").strip() + if not session_id: + result["skipped"] += 1 + continue + identity = "{}|{}".format(path.resolve(), digest) + event_id = "claudian_{}".format(hashlib.sha256(identity.encode()).hexdigest()[:24]) + if queue.enqueue({ + "event_id": event_id, "client": "claudian", + "event_type": "session_updated", "session_id": session_id, + "project_path": str(project_path) if project_path else str(root.parent.parent), + "source_ref": str(path.resolve()), "source_hash": digest, + }): + result["enqueued"] += 1 + stat = path.stat() + queue.update_checkpoint(str(path.resolve()), stat.st_ino, stat.st_size, digest) + return result + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("paths", nargs="*") + args = parser.parse_args(argv) + config = load_config() + paths = args.paths or list(config.claudian_session_dirs) + queue = CandidateEventQueue(config.database_path, config.max_candidate_events) + try: + result = import_sessions(paths, queue, project_path=config.knowledge_root) + finally: + queue.connection.close() + print(json.dumps(result, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/import_codex_rollouts.py b/scripts/import_codex_rollouts.py new file mode 100644 index 0000000..9f54bd3 --- /dev/null +++ b/scripts/import_codex_rollouts.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Incrementally import bounded pointers from append-only Codex rollouts.""" + +import argparse +import hashlib +import json +from pathlib import Path +import re +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from scripts.candidate_events import CandidateEventQueue +from scripts.bounded_proposals import ( + DEFAULT_MAX_PROPOSALS, DEFAULT_TAIL_BYTES, extract_proposals, +) +from scripts.config import load_config + + +SESSION_ID_PATTERN = re.compile( + r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" +) + + +def _fingerprint(path): + with Path(path).open("rb") as handle: + return hashlib.sha256(handle.readline(4096)).hexdigest() + + +def _event_type(record): + return "session_meta" if record.get("type") == "session_meta" else None + + +def _assistant_text(payload): + if payload.get("role") not in (None, "assistant"): + return "" + text = [] + for key in ("text", "output", "message"): + value = payload.get(key) + if isinstance(value, str): + text.append(value) + content = payload.get("content") + if isinstance(content, str): + text.append(content) + elif isinstance(content, list): + text.extend( + item.get("text", "") for item in content + if isinstance(item, dict) and item.get("type") == "output_text" + and isinstance(item.get("text"), str) + ) + return "\n".join(text) + + +def _import_tail_proposals(path, queue, session_id=None, project_path=None): + """Backfill explicit proposals once per changed transcript tail.""" + path = Path(path) + scan_ref = "codex-proposals:{}".format(path) + try: + with path.open("rb") as handle: + handle.seek(0, 2) + size = handle.tell() + handle.seek(max(0, size - DEFAULT_TAIL_BYTES)) + tail = handle.read(DEFAULT_TAIL_BYTES) + except OSError: + return 0 + digest = hashlib.sha256(tail).hexdigest() + checkpoint = queue.get_checkpoint(scan_ref) + if checkpoint and checkpoint[2] == digest: + return 0 + enqueued = 0 + proposal_count = 0 + base_offset = max(0, size - len(tail)) + relative_offset = 0 + for raw in tail.splitlines(True): + line_offset = base_offset + relative_offset + relative_offset += len(raw) + try: + record = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError): + continue + payload = record.get("payload", {}) + if record.get("type") == "session_meta": + session_id = payload.get("id") or payload.get("session_id") or session_id + project_path = payload.get("cwd") or project_path + if record.get("type") != "response_item": + continue + remaining = DEFAULT_MAX_PROPOSALS - proposal_count + if remaining <= 0: + break + for index, proposal in enumerate(extract_proposals( + _assistant_text(payload), max_proposals=remaining)): + proposal_hash = hashlib.sha256( + (str(path) + str(line_offset) + json.dumps(proposal, sort_keys=True, ensure_ascii=False)).encode() + ).hexdigest() + if queue.enqueue({ + "event_id": "codex_proposal_{}".format(proposal_hash[:24]), + "client": "codex", "event_type": "memory_proposal", + "session_id": session_id, "project_path": project_path, + "source_ref": "{}#tail-byte={}#proposal={}".format(path, line_offset, index), + "source_hash": proposal_hash, "excerpt": proposal["summary"], + "memory_type": proposal["type"], "confidence": proposal.get("confidence"), + "importance": proposal.get("importance"), + }): + enqueued += 1 + proposal_count += 1 + queue.update_checkpoint(scan_ref, path.stat().st_ino, size, digest) + return enqueued + + +def import_rollout(path, queue): + path = Path(path).resolve() + stat = path.stat() + fingerprint = _fingerprint(path) + checkpoint = queue.get_checkpoint(str(path)) + offset = 0 + if checkpoint: + old_inode, old_offset, old_hash = checkpoint + if old_inode == stat.st_ino and old_offset <= stat.st_size and old_hash == fingerprint: + offset = old_offset + + matches = SESSION_ID_PATTERN.findall(path.name) + session_id = matches[-1] if matches else None + project_path = None + processed = enqueued = 0 + next_offset = offset + with path.open("rb") as handle: + handle.seek(offset) + while True: + line_offset = handle.tell() + raw = handle.readline() + if not raw: + break + if not raw.endswith(b"\n"): + break + next_offset = handle.tell() + processed += 1 + try: + record = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError): + continue + payload = record.get("payload", {}) + if record.get("type") == "session_meta": + session_id = payload.get("id") or payload.get("session_id") or session_id + project_path = payload.get("cwd") or project_path + elif record.get("type") == "turn_context": + project_path = payload.get("cwd") or project_path + event_type = _event_type(record) + if not event_type: + continue + source_hash = hashlib.sha256(raw).hexdigest() + event_id = hashlib.sha256( + "{}|{}|{}|{}".format(path, stat.st_ino, line_offset, source_hash).encode() + ).hexdigest() + if queue.enqueue({ + "event_id": "codex_{}".format(event_id[:24]), + "client": "codex", + "event_type": event_type, + "session_id": session_id, + "project_path": project_path, + "source_ref": "{}#byte={}".format(path, line_offset), + "source_hash": source_hash, + }): + enqueued += 1 + queue.update_checkpoint(str(path), stat.st_ino, next_offset, fingerprint) + enqueued += _import_tail_proposals(path, queue, session_id, project_path) + return {"processed": processed, "enqueued": enqueued} + + +def _rollouts(paths): + for raw_path in paths: + path = Path(raw_path).expanduser() + if path.is_file() and path.suffix == ".jsonl": + yield path + elif path.is_dir(): + yield from path.rglob("*.jsonl") + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("paths", nargs="*") + args = parser.parse_args(argv) + paths = args.paths or ["~/.codex/sessions", "~/.codex/archived_sessions"] + config = load_config() + queue = CandidateEventQueue(config.database_path, config.max_candidate_events) + total = {"files": 0, "processed": 0, "enqueued": 0} + try: + for path in _rollouts(paths): + result = import_rollout(path, queue) + total["files"] += 1 + total["processed"] += result["processed"] + total["enqueued"] += result["enqueued"] + finally: + queue.connection.close() + print(json.dumps(total, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/import_hermes_sessions.py b/scripts/import_hermes_sessions.py new file mode 100644 index 0000000..824026b --- /dev/null +++ b/scripts/import_hermes_sessions.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Capture bounded, incremental pointers from Hermes state.db.""" + +import argparse +import hashlib +import json +from pathlib import Path +import sqlite3 +import sys +from urllib.parse import quote + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from scripts.candidate_events import CandidateEventQueue +from scripts.bounded_proposals import DEFAULT_MAX_PROPOSALS, extract_proposals +from scripts.config import load_config + + +REQUIRED = { + "sessions": {"id", "source", "cwd", "message_count"}, + "messages": {"id", "session_id"}, +} + + +def _connect(path): + uri = "file:{}?mode=ro".format(quote(str(path.resolve()), safe="/")) + connection = sqlite3.connect(uri, uri=True) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA query_only=ON") + return connection + + +def _validate_schema(connection): + for table, required in REQUIRED.items(): + exists = connection.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,) + ).fetchone() + columns = { + row[1] for row in connection.execute( + "PRAGMA table_info({})".format(table) + ) + } + if not exists or not required.issubset(columns): + raise ValueError("required Hermes schema missing: {}".format(table)) + + +def import_sessions(path, queue): + path = Path(path).expanduser().resolve() + connection = _connect(path) + try: + _validate_schema(connection) + rows = connection.execute(""" + SELECT s.id, s.source, s.cwd, s.message_count, + COALESCE(MAX(m.id), 0) AS max_message_id + FROM sessions s + LEFT JOIN messages m ON m.session_id=s.id + GROUP BY s.id, s.source, s.cwd, s.message_count + ORDER BY s.id + """).fetchall() + proposals = {} + for row in rows: + messages = connection.execute( + "SELECT id, content FROM messages WHERE session_id=? AND role='assistant' " + "AND content IS NOT NULL ORDER BY id DESC LIMIT 8", (row["id"],) + ).fetchall() + proposals[row["id"]] = [(message["id"], proposal) + for message in messages + for proposal in extract_proposals(message["content"])][ + :DEFAULT_MAX_PROPOSALS + ] + finally: + connection.close() + + result = {"sessions": len(rows), "enqueued": 0, "skipped": 0} + stat = path.stat() + for row in rows: + session_id = str(row["id"]) + digest = hashlib.sha256(json.dumps({ + "id": session_id, + "source": row["source"], + "cwd": row["cwd"], + "message_count": row["message_count"], + "max_message_id": row["max_message_id"], + }, sort_keys=True, ensure_ascii=False).encode()).hexdigest() + checkpoint_ref = "hermes:{}#{}".format(path, session_id) + checkpoint = queue.get_checkpoint(checkpoint_ref) + session_changed = not ( + checkpoint and checkpoint[1] == int(row["max_message_id"]) and checkpoint[2] == digest + ) + if not session_changed: + result["skipped"] += 1 + source_ref = "{}#session={}&max_message_id={}".format( + path, session_id, row["max_message_id"] + ) + event_id = "hermes_{}".format( + hashlib.sha256((source_ref + "|" + digest).encode()).hexdigest()[:24] + ) + if session_changed and queue.enqueue({ + "event_id": event_id, + "client": "hermes", + "event_type": "session_updated", + "session_id": session_id, + "project_path": row["cwd"], + "source_ref": source_ref, + "source_hash": digest, + }): + result["enqueued"] += 1 + for message_id, proposal in proposals.get(row["id"], []): + proposal_hash = hashlib.sha256( + (source_ref + str(message_id) + proposal["summary"]).encode() + ).hexdigest() + if queue.enqueue({ + "event_id": "hermes_proposal_{}".format(proposal_hash[:24]), + "client": "hermes", + "event_type": "memory_proposal", + "session_id": session_id, + "project_path": row["cwd"], + "source_ref": "{}&message_id={}".format(source_ref, message_id), + "source_hash": proposal_hash, + "excerpt": proposal["summary"], + "memory_type": proposal["type"], + "confidence": proposal.get("confidence"), + "importance": proposal.get("importance"), + }): + result["enqueued"] += 1 + queue.update_checkpoint(checkpoint_ref, stat.st_ino, row["max_message_id"], digest) + return result + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("path", nargs="?") + args = parser.parse_args(argv) + config = load_config() + path = args.path or config.hermes_state_db + if not path: + parser.error("Hermes state database is not configured") + queue = CandidateEventQueue(config.database_path, config.max_candidate_events) + try: + result = import_sessions(path, queue) + except (OSError, sqlite3.Error, ValueError) as error: + print(json.dumps({"error": str(error)}, ensure_ascii=False), file=sys.stderr) + return 2 + finally: + queue.connection.close() + print(json.dumps(result, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/import_native_history.py b/scripts/import_native_history.py new file mode 100644 index 0000000..bb7e165 --- /dev/null +++ b/scripts/import_native_history.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Bounded automatic importer for Codex, Claude, Cowork and Hermes history.""" + +import argparse +from dataclasses import dataclass +from fnmatch import fnmatch +import json +import os +from pathlib import Path +import sys +import time + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from scripts.candidate_events import CandidateEventQueue +from scripts.config import load_config +from scripts.native_auto_extract import import_native_file + + +@dataclass(frozen=True) +class SourceSpec: + roots: tuple + patterns: tuple + prefer_jsonl_mirror: bool = False + + +def _has_jsonl_mirror(path): + if not path.name.startswith("session_") or path.suffix != ".json": + return False + return path.with_name(path.name[len("session_"):-len(".json")] + ".jsonl").is_file() + + +def _files(spec): + for raw in spec.roots: + path = Path(raw).expanduser() + if path.is_file() and any(fnmatch(path.name, pattern) for pattern in spec.patterns): + yield path + elif path.is_dir(): + for candidate in path.rglob("*"): + if not candidate.is_file(): + continue + if not any(fnmatch(candidate.name, pattern) for pattern in spec.patterns): + continue + if spec.prefer_jsonl_mirror and _has_jsonl_mirror(candidate): + continue + yield candidate + + +def _source_spec(client, value): + if isinstance(value, SourceSpec): + return value + patterns = ("*.jsonl",) if client == "claude-desktop" else ("*.json", "*.jsonl") + return SourceSpec(tuple(value), patterns) + + +def source_files(paths_by_client): + return { + client: sorted(set(_files(_source_spec(client, value)))) + for client, value in paths_by_client.items() + } + + +def _matches_spec(path, spec): + path = Path(path) + if not any(fnmatch(path.name, pattern) for pattern in spec.patterns): + return False + if spec.prefer_jsonl_mirror and _has_jsonl_mirror(path): + return False + return True + + +def prune_obsolete_sources(paths_by_client, queue): + specs = { + client: _source_spec(client, value) + for client, value in paths_by_client.items() + } + rows = queue.connection.execute( + "SELECT source_ref FROM import_checkpoints WHERE source_ref LIKE 'native-auto:%'" + ).fetchall() + obsolete = [] + for row in rows: + checkpoint_ref = row[0] + try: + _, client, raw_path = checkpoint_ref.split(":", 2) + except ValueError: + continue + spec = specs.get(client) + if spec and not _matches_spec(raw_path, spec): + obsolete.append((checkpoint_ref, client, raw_path)) + events = 0 + for checkpoint_ref, client, raw_path in obsolete: + cursor = queue.connection.execute( + "DELETE FROM candidate_events WHERE client=? AND event_id LIKE 'auto_%' " + "AND (source_ref=? OR source_ref LIKE ?)", + (client, raw_path, raw_path + "#%"), + ) + events += cursor.rowcount + queue.connection.execute( + "DELETE FROM import_checkpoints WHERE source_ref=?", (checkpoint_ref,) + ) + queue.connection.commit() + return {"checkpoints": len(obsolete), "events": events} + + +def import_paths(paths_by_client, queue, max_files=40, max_bytes=64 * 1024, min_age_seconds=120): + total = {"files": 0, "processed": 0, "enqueued": 0, "proposals": 0} + for client, paths in source_files(paths_by_client).items(): + paths = sorted(paths, key=lambda p: p.stat().st_mtime, reverse=True) + pending = [] + for path in paths: + stat = path.stat() + if time.time() - stat.st_mtime < int(min_age_seconds): + continue + checkpoint = queue.get_checkpoint( + "native-auto:{}:{}".format(client, path.expanduser().resolve()) + ) + if not checkpoint or checkpoint[0] != stat.st_ino or checkpoint[1] != stat.st_size: + pending.append(path) + for path in pending[:int(max_files)]: + try: + result = import_native_file(path, queue, client=client, max_bytes=max_bytes) + except (OSError, ValueError): + continue + total["files"] += 1 + total["processed"] += int(bool(result["processed"])) + total["enqueued"] += result["enqueued"] + total["proposals"] += result["proposals"] + return total + + +def configured_paths(config): + return { + "codex": SourceSpec( + ("~/.codex/sessions", "~/.codex/archived_sessions"), ("*.jsonl",) + ), + "claude-code": SourceSpec(("~/.claude/projects",), ("*.jsonl",)), + "claude-desktop": SourceSpec( + tuple(config.claude_desktop_session_dirs), ("audit.jsonl",) + ), + "hermes": SourceSpec( + ("~/.hermes/sessions",), ("*.jsonl", "session_*.json"), + prefer_jsonl_mirror=True, + ), + } + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--max-files", type=int, default=int(os.getenv("DNA_MEMORY_MAX_FILES", "40"))) + parser.add_argument("--max-bytes", type=int, default=64 * 1024) + parser.add_argument("--prune-obsolete", action="store_true") + args = parser.parse_args(argv) + config = load_config() + queue = CandidateEventQueue(config.database_path, config.max_candidate_events) + try: + paths = configured_paths(config) + result = import_paths(paths, queue, args.max_files, args.max_bytes) + if args.prune_obsolete: + result["pruned"] = prune_obsolete_sources(paths, queue) + finally: + queue.connection.close() + print(json.dumps(result, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/markdown_memory.py b/scripts/markdown_memory.py new file mode 100644 index 0000000..65f8865 --- /dev/null +++ b/scripts/markdown_memory.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Adapter for explicitly managed Markdown memory pages.""" + +import hashlib +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Optional, Tuple + +import yaml + +from scripts.policy import inspect_content + + +SUPPORTED_TYPES = { + "preference", "decision", "fact", "insight", "workflow", + "error_lesson", "project_state", "open_loop", +} + + +@dataclass(frozen=True) +class ReindexResult: + scanned: int = 0 + indexed: int = 0 + skipped: int = 0 + removed: int = 0 + + +def _parse(path: Path) -> Optional[Dict[str, object]]: + text = path.read_text(encoding="utf-8") + if not text.startswith("---\n"): + return None + try: + raw, body = text[4:].split("\n---\n", 1) + meta = yaml.safe_load(raw) or {} + except (ValueError, yaml.YAMLError): + return None + tags = meta.get("tags", []) + if isinstance(tags, str): + tags = [tags] + if not meta.get("id") or meta.get("type") not in SUPPORTED_TYPES or "memory" not in tags: + return None + if meta.get("status", "active") not in {"active", "superseded", "archived", "rejected"}: + return None + supersedes = meta.get("supersedes", []) + if supersedes is None: + supersedes = [] + if not isinstance(supersedes, list) or not all( + isinstance(item, str) and item.strip() for item in supersedes + ): + return None + superseded_by = meta.get("superseded_by") + if superseded_by is not None and ( + not isinstance(superseded_by, str) or not superseded_by.strip() + ): + return None + policy = inspect_content(text) + if not policy.allowed: + return None + paragraphs = [part.strip() for part in body.split("\n\n") if part.strip()] + if not paragraphs: + return None + digest = hashlib.sha256(text.encode()).hexdigest() + return { + "memory_id": str(meta["id"]), + "markdown_path": str(path.resolve()), + "type": meta["type"], + "status": meta.get("status", "active"), + "summary": " ".join(paragraphs[0].split()), + "content_hash": digest, + "source_hash": meta.get("source_hash"), + "source_refs": meta.get("source_refs", []), + "supersedes": supersedes, + "superseded_by": superseded_by, + "confidence": meta.get("confidence"), + "importance": float(meta.get("importance", 0.5)), + "sensitivity": meta.get("sensitivity", "normal"), + "clients": meta.get("clients", []), + "project_path": meta.get("project_path"), + "session_id": meta.get("session_id"), + "created_at": str(meta.get("created", "")), + "updated_at": str(meta.get("updated", "")), + "expires_at": str(meta.get("expires", "")), + } + + +def reindex_markdown(root: Path, store) -> ReindexResult: + root = Path(root) + scanned = indexed = skipped = 0 + present = set() + if root.exists(): + for path in root.rglob("*.md"): + scanned += 1 + record = _parse(path) + if record is None: + skipped += 1 + continue + present.add(record["memory_id"]) + if store.upsert_markdown(record): + indexed += 1 + removed = store.remove_missing_markdown(present) + store.commit() + return ReindexResult(scanned, indexed, skipped, removed) diff --git a/scripts/memory_cli.py b/scripts/memory_cli.py new file mode 100644 index 0000000..9fc2bbb --- /dev/null +++ b/scripts/memory_cli.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""CLI for rebuildable long-term memory index operations.""" + +import argparse +import json +from pathlib import Path +from typing import Optional, Sequence + +from scripts.config import load_config +from scripts.client_coverage import build_coverage_report +from scripts.markdown_memory import reindex_markdown +from scripts.memory_operations import MemoryOperations +from scripts.policy import capacity_status +from scripts.unified_memory import UnifiedMemoryStore +from scripts.memory_value import memory_value + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="dna memory") + parser.add_argument("--profile", type=Path) + sub = parser.add_subparsers(dest="command", required=True) + for name in ("status", "reindex", "coverage"): + command = sub.add_parser(name) + command.add_argument("--json", action="store_true", dest="as_json") + maintain = sub.add_parser("maintain") + maintain.add_argument("level", choices=("daily", "weekly", "monthly")) + maintain.add_argument("--json", action="store_true", dest="as_json") + maintain.add_argument("--now") + maintain.add_argument("--backup-stamp") + value = sub.add_parser("value") + value.add_argument("--json", action="store_true", dest="as_json") + value.add_argument("--now") + return parser + + +def _emit(payload, as_json): + if as_json: + print(json.dumps(payload, ensure_ascii=False, sort_keys=True)) + else: + for key, value in payload.items(): + print("{}: {}".format(key, value)) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + args = _parser().parse_args(argv) + config = load_config(args.profile) + if args.command == "coverage": + _emit(build_coverage_report(config), args.as_json) + return 0 + if args.command == "value": + _emit(memory_value(config, now=args.now), args.as_json) + return 0 + if args.command == "maintain": + operation = getattr(MemoryOperations(config), args.level) + kwargs = {"now": args.now} + if args.level != "daily": + kwargs["backup_stamp"] = args.backup_stamp + _emit(operation(**kwargs), args.as_json) + return 0 + preflight = capacity_status(config.database_path, config.warning_bytes, config.hard_bytes) + if args.command == "reindex" and not preflight.writable: + _emit({ + "error": "capacity_blocked", + "size_bytes": preflight.size_bytes, + "max_records": config.max_records, + }, args.as_json) + return 2 + store = UnifiedMemoryStore(config.database_path) + try: + record_count = store.connection.execute("SELECT COUNT(*) FROM memory_index").fetchone()[0] + if args.command == "reindex": + cap = capacity_status(config.database_path, config.warning_bytes, config.hard_bytes) + if not cap.writable: + _emit({ + "error": "capacity_blocked", + "size_bytes": cap.size_bytes, + "record_count": record_count, + "max_records": config.max_records, + }, args.as_json) + return 2 + result = reindex_markdown(config.knowledge_root, store) + _emit({ + "scanned": result.scanned, "indexed": result.indexed, + "skipped": result.skipped, "removed": result.removed, + }, args.as_json) + return 0 + cap = capacity_status(config.database_path, config.warning_bytes, config.hard_bytes) + legacy = store.connection.execute( + "SELECT COUNT(*) FROM memory_index WHERE source_kind='legacy_cache'" + ).fetchone()[0] + payload = { + "truth_root": str(config.knowledge_root), + "truth_root_exists": config.knowledge_root.is_dir(), + "database_path": str(config.database_path), + "managed_records": store.count_managed(), + "legacy_records": legacy, + "capacity": { + "state": "warning" if record_count >= config.max_records and cap.state == "ok" else cap.state, + "size_bytes": cap.size_bytes, + "writable": cap.writable, + "record_count": record_count, + "max_records": config.max_records, + }, + } + _emit(payload, args.as_json) + return 0 + finally: + store.close() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/memory_mcp.py b/scripts/memory_mcp.py new file mode 100644 index 0000000..d394e58 --- /dev/null +++ b/scripts/memory_mcp.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Thin stdio MCP adapter for the Markdown-backed memory service.""" + +import dataclasses +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from mcp.server.fastmcp import FastMCP + +from scripts.config import load_config +from scripts.memory_service import MemoryService, MemoryValidationError + + +def _error(error): + code = "validation_error" if isinstance(error, (MemoryValidationError, ValueError)) else "internal_error" + if code == "internal_error": + print("dna-memory MCP error: {}".format(error), file=sys.stderr) + return {"ok": False, "error": {"code": code, "message": str(error)}} + + +def build_server(service): + mcp = FastMCP("dna-memory") + + @mcp.tool() + def memory_recall( + query: str, limit: int = 20, client: str = None, session_id: str = None + ): + """Recall active memories whose summaries match a query.""" + try: + return { + "ok": True, + "memories": service.recall(query, limit, client, session_id), + } + except Exception as error: + return _error(error) + + @mcp.tool() + def memory_get(memory_id: str): + """Get one indexed memory by its stable ID.""" + try: + return {"ok": True, "memory": service.get(memory_id)} + except Exception as error: + return _error(error) + + @mcp.tool() + def memory_remember( + type: str, summary: str, source_hash: str = None, + confidence: str = "medium", importance: float = 0.5, + source_ref: str = None, client: str = None, + project_path: str = None, session_id: str = None, + supersedes: list[str] = None, + ): + """Write a reviewed memory proposal to the Markdown truth store.""" + try: + result = service.remember({ + "type": type, "summary": summary, "source_hash": source_hash, + "confidence": confidence, "importance": importance, + "source_refs": [source_ref] if source_ref else [], + "clients": [client] if client else [], + "project_path": project_path, "session_id": session_id, + "supersedes": supersedes or [], + }) + return {"ok": True, **result} + except Exception as error: + return _error(error) + + @mcp.tool() + def memory_feedback( + memory_id: str, outcome: str, note: str = None, + client: str = None, session_id: str = None, + ): + """Record whether a recalled memory was useful or misleading.""" + try: + result = service.feedback(memory_id, outcome, note, client, session_id) + return {"ok": True, **result} + except Exception as error: + return _error(error) + + @mcp.tool() + def memory_close_session( + client: str, session_id: str, project_path: str = None, + source_ref: str = None, + ): + """Queue bounded session metadata without copying the transcript.""" + try: + result = service.close_session(client, session_id, project_path, source_ref) + return {"ok": True, **result} + except Exception as error: + return _error(error) + + @mcp.tool() + def memory_status(): + """Return current memory store status.""" + try: + return {"ok": True, **service.status()} + except Exception as error: + return _error(error) + + @mcp.tool() + def memory_reindex(): + """Rebuild the disposable SQLite index from Markdown truth.""" + try: + result = service.reindex() + return {"ok": True, **dataclasses.asdict(result)} + except Exception as error: + return _error(error) + + return mcp + + +def main(): + build_server(MemoryService(load_config())).run(transport="stdio") + + +if __name__ == "__main__": + main() diff --git a/scripts/memory_operations.py b/scripts/memory_operations.py new file mode 100644 index 0000000..fd6762a --- /dev/null +++ b/scripts/memory_operations.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Bounded crystallization, retention, backup, and integrity operations.""" + +from datetime import datetime +from pathlib import Path +import sqlite3 + +from scripts.candidate_events import CandidateEventQueue +from scripts.markdown_memory import SUPPORTED_TYPES, reindex_markdown +from scripts.memory_service import MemoryService +from scripts.policy import inspect_content + + +class MemoryOperations: + def __init__(self, config): + self.config = config + + def daily(self, now=None): + now = now or datetime.now().strftime("%Y-%m-%d %H:%M:%S") + queue = CandidateEventQueue(self.config.database_path, self.config.max_candidate_events) + result = { + "crystallized": 0, "rejected": 0, "compacted": 0, + "expired": 0, "deleted": 0, + } + try: + proposals = queue.connection.execute(""" + SELECT event_id, client, session_id, project_path, source_ref, + source_hash, excerpt, memory_type, confidence, importance + FROM candidate_events + WHERE status='pending' AND event_type='memory_proposal' + ORDER BY created_at, event_id + """).fetchall() + service = MemoryService(self.config) + try: + for row in proposals: + event_id, client, session_id, project_path, source_ref, source_hash, summary, mem_type, confidence, importance = row + error = self._proposal_error(mem_type, summary) + if error: + queue.connection.execute( + "UPDATE candidate_events SET status='rejected', processed_at=?, error=? WHERE event_id=?", + (now, error, event_id), + ) + queue.connection.commit() + result["rejected"] += 1 + continue + remembered = service.remember({ + "type": mem_type, "summary": summary, + "source_hash": source_hash or "event:{}".format(event_id), + "confidence": confidence or "medium", + "importance": importance if importance is not None else 0.5, + "source_refs": [source_ref] if source_ref else [], + "clients": [client], "project_path": project_path, + "session_id": session_id, + }) + queue.connection.execute( + "UPDATE candidate_events SET status='crystallized', processed_at=?, memory_id=?, error=NULL WHERE event_id=?", + (now, remembered["memory_id"], event_id), + ) + queue.connection.commit() + result["crystallized"] += 1 + finally: + service.store.close() + + cursor = queue.connection.execute(""" + UPDATE candidate_events AS event + SET status='superseded', processed_at=?, + error='covered by codex session_meta pointer' + WHERE event.status='pending' AND event.client='codex' + AND event.event_type NOT IN ('memory_proposal', 'session_meta') + AND event.session_id IS NOT NULL + AND EXISTS ( + SELECT 1 FROM candidate_events AS session + WHERE session.client='codex' + AND session.event_type='session_meta' + AND session.session_id=event.session_id + ) + """, (now,)) + result["compacted"] = cursor.rowcount + cursor = queue.connection.execute(""" + UPDATE candidate_events + SET status='expired', processed_at=? + WHERE status='pending' AND event_type!='memory_proposal' + AND datetime(created_at) < datetime(?, '-30 days') + """, (now, now)) + result["expired"] = cursor.rowcount + cursor = queue.connection.execute(""" + DELETE FROM candidate_events + WHERE status IN ('crystallized', 'rejected', 'superseded', 'expired') + AND datetime(COALESCE(processed_at, created_at)) < datetime(?, '-7 days') + """, (now,)) + result["deleted"] = cursor.rowcount + queue.connection.commit() + return result + finally: + queue.connection.close() + + @staticmethod + def _proposal_error(mem_type, summary): + if mem_type not in SUPPORTED_TYPES: + return "invalid memory type" + if not summary or not str(summary).strip(): + return "missing safe summary" + policy = inspect_content(str(summary)) + if not policy.allowed: + return "sensitive summary rejected: {}".format(policy.reason) + return None + + def weekly(self, now=None, backup_stamp=None): + result = self.daily(now=now) + stamp = backup_stamp or datetime.now().strftime("%Y%m%dT%H%M%S") + self.config.backup_dir.mkdir(parents=True, exist_ok=True) + target = self.config.backup_dir / "memory-{}.db".format(stamp) + source = sqlite3.connect(str(self.config.database_path)) + backup = sqlite3.connect(str(target)) + try: + source.backup(backup) + finally: + backup.close() + source.close() + backups = sorted(self.config.backup_dir.glob("memory-*.db")) + for old in backups[:-max(1, self.config.backup_keep)]: + old.unlink() + connection = sqlite3.connect(str(self.config.database_path)) + try: + connection.execute("VACUUM") + finally: + connection.close() + result.update({"backup_path": str(target), "backups_kept": len(list( + self.config.backup_dir.glob("memory-*.db") + ))}) + return result + + def monthly(self, now=None, backup_stamp=None): + result = self.weekly(now=now, backup_stamp=backup_stamp) + connection = sqlite3.connect(str(self.config.database_path)) + try: + integrity = connection.execute("PRAGMA integrity_check").fetchone()[0] + finally: + connection.close() + service = MemoryService(self.config) + try: + rebuilt = reindex_markdown(self.config.knowledge_root, service.store) + finally: + service.store.close() + result["integrity"] = integrity + result["reindex"] = { + "scanned": rebuilt.scanned, "indexed": rebuilt.indexed, + "skipped": rebuilt.skipped, "removed": rebuilt.removed, + } + return result diff --git a/scripts/memory_service.py b/scripts/memory_service.py new file mode 100644 index 0000000..b361ba9 --- /dev/null +++ b/scripts/memory_service.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +"""Business API for Markdown-backed long-term memory.""" + +import os +import time +import uuid +import hashlib +import json +import re +import shutil +from pathlib import Path + +import yaml + +from scripts.markdown_memory import SUPPORTED_TYPES, reindex_markdown +from scripts.candidate_events import CandidateEventQueue +from scripts.policy import inspect_content +from scripts.unified_memory import UnifiedMemoryStore + + +class MemoryValidationError(ValueError): + pass + + +class MemoryService: + QUERY_TERM_RE = re.compile(r"[^\W_]+", re.UNICODE) + + def __init__(self, config): + self.config = config + self.store = UnifiedMemoryStore(config.database_path) + + def remember(self, proposal): + summary = str(proposal.get("summary", "")).strip() + mem_type = proposal.get("type") + if mem_type not in SUPPORTED_TYPES or not summary: + raise MemoryValidationError("type and summary are required") + policy = inspect_content(summary) + if not policy.allowed: + raise MemoryValidationError("sensitive content rejected: {}".format(policy.reason)) + source_hash = proposal.get("source_hash") + if source_hash: + row = self.store.connection.execute( + "SELECT memory_id FROM memory_index WHERE source_hash=?", (source_hash,) + ).fetchone() + if row: + return {"created": False, "memory_id": row[0], "superseded": []} + supersedes = self._normalize_supersedes(proposal.get("supersedes")) + root = self.config.knowledge_root / self.config.managed_memory_dir + targets = self._load_active_targets(supersedes, root) + memory_id = "mem_{}".format(uuid.uuid4().hex) + now = time.strftime("%Y-%m-%dT%H:%M:%S%z") + meta = { + "id": memory_id, "type": mem_type, "status": "active", + "confidence": proposal.get("confidence", "medium"), + "importance": float(proposal.get("importance", 0.5)), + "created": now, "updated": now, + "clients": proposal.get("clients", []), + "source_hash": source_hash, + "source_refs": proposal.get("source_refs", []), + "supersedes": supersedes, + "tags": ["memory", mem_type], + "project_path": proposal.get("project_path"), + "session_id": proposal.get("session_id"), + } + root.mkdir(parents=True, exist_ok=True) + path = root / "{}.md".format(memory_id) + replacements = {} + for target in targets: + old_meta = dict(target["meta"]) + old_meta["status"] = "superseded" + old_meta["superseded_by"] = memory_id + old_meta["updated"] = now + replacements[target["path"]] = self._render_memory( + old_meta, target["body"] + ) + replacements[path] = self._render_memory(meta, summary) + self._install_memory_files(replacements) + return { + "created": True, "memory_id": memory_id, "path": str(path), + "superseded": supersedes, + } + + @staticmethod + def _normalize_supersedes(value): + if value is None: + return [] + if not isinstance(value, list): + raise MemoryValidationError("supersedes must be a list of memory IDs") + result = [] + for item in value: + if not isinstance(item, str) or not item.strip(): + raise MemoryValidationError( + "supersedes must contain non-empty memory IDs" + ) + memory_id = item.strip() + if memory_id not in result: + result.append(memory_id) + return result + + def _load_active_targets(self, memory_ids, root): + root = Path(root).resolve() + targets = [] + for memory_id in memory_ids: + row = self.store.connection.execute( + "SELECT memory_id, markdown_path, status, source_kind " + "FROM memory_index WHERE memory_id=?", (memory_id,) + ).fetchone() + if not row: + raise MemoryValidationError( + "superseded memory not found: {}".format(memory_id) + ) + if row["source_kind"] != "markdown": + raise MemoryValidationError( + "superseded memory is not Markdown-managed: {}".format(memory_id) + ) + if row["status"] != "active": + raise MemoryValidationError( + "superseded memory is not active: {}".format(memory_id) + ) + path = Path(row["markdown_path"] or "").resolve() + try: + path.relative_to(root) + except ValueError: + raise MemoryValidationError( + "superseded memory is outside managed root: {}".format(memory_id) + ) + try: + text = path.read_text(encoding="utf-8") + raw, body = text[4:].split("\n---\n", 1) + meta = yaml.safe_load(raw) or {} + except (OSError, ValueError, yaml.YAMLError): + raise MemoryValidationError( + "superseded memory Markdown is invalid: {}".format(memory_id) + ) + if not text.startswith("---\n") or meta.get("id") != memory_id: + raise MemoryValidationError( + "superseded memory Markdown ID mismatch: {}".format(memory_id) + ) + targets.append({"path": path, "meta": meta, "body": body}) + return targets + + @staticmethod + def _render_memory(meta, body): + return "---\n{}---\n\n{}\n".format( + yaml.safe_dump(meta, allow_unicode=True, sort_keys=False), + str(body).strip("\n"), + ) + + def _install_memory_files(self, replacements): + transaction_id = uuid.uuid4().hex + staged = {} + backups = {} + installed = [] + try: + for path, content in replacements.items(): + path.parent.mkdir(parents=True, exist_ok=True) + temp = path.with_name(".{}.{}.tmp".format(path.name, transaction_id)) + temp.write_text(content, encoding="utf-8") + staged[path] = temp + for path in replacements: + if path.exists(): + backup = path.with_name( + ".{}.{}.bak".format(path.name, transaction_id) + ) + shutil.copy2(str(path), str(backup)) + backups[path] = backup + for path, temp in staged.items(): + os.replace(str(temp), str(path)) + installed.append(path) + reindex_markdown(self.config.knowledge_root, self.store) + except Exception: + rollback_error = None + for path in reversed(installed): + backup = backups.get(path) + try: + if backup and backup.exists(): + os.replace(str(backup), str(path)) + elif path.exists(): + path.unlink() + except OSError as error: + rollback_error = rollback_error or error + for path, backup in backups.items(): + if path not in installed and backup.exists(): + backup.unlink() + for temp in staged.values(): + if temp.exists(): + temp.unlink() + if rollback_error is None: + reindex_markdown(self.config.knowledge_root, self.store) + else: + raise RuntimeError("memory file rollback failed") from rollback_error + raise + else: + for backup in backups.values(): + if backup.exists(): + backup.unlink() + + @classmethod + def _query_terms(cls, query): + terms = [] + seen = set() + for match in cls.QUERY_TERM_RE.findall(str(query or "")): + term = match.casefold() + if term and term not in seen: + terms.append(term) + seen.add(term) + return terms + + def recall(self, query, limit=20, client=None, session_id=None): + limit = max(1, min(int(limit), 20)) + terms = self._query_terms(query) + if not terms: + raise MemoryValidationError("query is required") + patterns = ["%{}%".format(term) for term in terms] + hit_count = " + ".join( + "CASE WHEN lower(m.summary) LIKE ? THEN 1 ELSE 0 END" for _ in terms + ) + matches = " OR ".join("lower(m.summary) LIKE ?" for _ in terms) + sql = """ + WITH feedback_scores AS ( + SELECT memory_id, + SUM(CASE outcome + WHEN 'useful' THEN 1 + WHEN 'misleading' THEN -1 + ELSE 0 END) AS feedback_score + FROM memory_feedback + GROUP BY memory_id + ), ranked AS ( + SELECT m.memory_id, m.type, m.status, m.summary, m.importance, + m.confidence, m.markdown_path, m.source_refs, m.clients, + m.updated_at, ({hit_count}) AS hit_count, + COALESCE(f.feedback_score, 0) AS feedback_score, + CASE lower(COALESCE(m.confidence, '')) + WHEN 'high' THEN 3 + WHEN 'medium' THEN 2 + WHEN 'low' THEN 1 + ELSE 0 + END AS confidence_rank + FROM memory_index m + LEFT JOIN feedback_scores f ON f.memory_id=m.memory_id + WHERE m.status='active' AND ({matches}) + ) + SELECT memory_id, type, status, summary, importance, confidence, + markdown_path, source_refs, clients + FROM ranked + ORDER BY hit_count DESC, feedback_score DESC, confidence_rank DESC, + importance DESC, updated_at DESC + LIMIT ? + """.format(hit_count=hit_count, matches=matches) + rows = self.store.connection.execute( + sql, tuple(patterns + patterns + [limit]) + ).fetchall() + self.store.connection.execute( + "INSERT INTO memory_recall_events " + "(query_hash, client, session_id, result_count) VALUES (?, ?, ?, ?)", + ( + hashlib.sha256(str(query).encode()).hexdigest(), + client, + session_id, + len(rows), + ), + ) + if rows: + self.store.connection.executemany( + "UPDATE memory_index " + "SET recall_count=recall_count+1, last_recalled_at=CURRENT_TIMESTAMP " + "WHERE memory_id=?", + ((row["memory_id"],) for row in rows), + ) + self.store.connection.commit() + return [self._decode_record(row) for row in rows] + + def get(self, memory_id): + row = self.store.connection.execute( + "SELECT * FROM memory_index WHERE memory_id=?", (memory_id,) + ).fetchone() + return self._decode_record(row) if row else None + + @staticmethod + def _decode_record(row): + record = dict(row) + for field in ("source_refs", "clients", "supersedes"): + value = record.get(field) + if isinstance(value, str): + try: + record[field] = json.loads(value) + except json.JSONDecodeError: + record[field] = [] + return record + + def feedback(self, memory_id, outcome, note=None, client=None, session_id=None): + if not self.get(memory_id): + raise MemoryValidationError("memory not found") + outcome = str(outcome).strip() + if not outcome: + raise MemoryValidationError("outcome is required") + if note and not inspect_content(str(note)).allowed: + raise MemoryValidationError("sensitive content rejected") + self.store.connection.execute( + "INSERT INTO memory_feedback " + "(memory_id, outcome, note, client, session_id) VALUES (?, ?, ?, ?, ?)", + (memory_id, outcome, note, client, session_id), + ) + self.store.connection.commit() + return {"recorded": True} + + def close_session(self, client, session_id, project_path=None, source_ref=None): + if not str(client).strip() or not str(session_id).strip(): + raise MemoryValidationError("client and session_id are required") + identity = "|".join((str(client), str(session_id), str(source_ref or ""))) + event_id = "close_{}".format(hashlib.sha256(identity.encode()).hexdigest()[:24]) + queue = CandidateEventQueue(self.config.database_path, self.config.max_candidate_events) + queue.enqueue({ + "event_id": event_id, + "client": str(client), + "event_type": "session_closed", + "session_id": str(session_id), + "project_path": project_path, + "source_ref": source_ref, + }) + queue.connection.close() + return {"event_id": event_id} + + def reindex(self): + return reindex_markdown(self.config.knowledge_root, self.store) + + def status(self): + return {"managed_records": self.store.count_managed(), "truth_root": str(self.config.knowledge_root)} diff --git a/scripts/memory_value.py b/scripts/memory_value.py new file mode 100644 index 0000000..e859d1f --- /dev/null +++ b/scripts/memory_value.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Privacy-bounded value metrics for the cross-client memory loop.""" + +import json +import sqlite3 +from datetime import datetime +from urllib.parse import quote + + +CLIENT_FIELDS = ( + "candidate_events", "recall_attempts", "returned_memories", + "useful", "misleading", "new_memories", +) + + +def _family(client): + value = str(client or "unknown").strip().casefold() + if value.startswith("codex"): + return "codex" + if value.startswith("claude") or value.startswith("claudian"): + return "claude" + if value.startswith("hermes"): + return "hermes" + return value or "unknown" + + +def _table_exists(connection, table): + return connection.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,) + ).fetchone() is not None + + +def _window_clause(column, days, now): + if days is None: + return "", () + return ( + " WHERE datetime({}) >= datetime(?, ?) " + "AND datetime({}) <= datetime(?)".format(column, column), + (now, "-{} days".format(days), now), + ) + + +def _metrics(connection, days, now): + metrics = { + "recall_attempts": 0, "recall_hits": 0, "hit_rate": 0.0, + "returned_memories": 0, "useful": 0, "misleading": 0, + "unfeedback": 0, "new_memories": 0, + } + if _table_exists(connection, "memory_recall_events"): + clause, params = _window_clause("created_at", days, now) + row = connection.execute( + "SELECT COUNT(*), " + "COALESCE(SUM(CASE WHEN result_count > 0 THEN 1 ELSE 0 END), 0), " + "COALESCE(SUM(result_count), 0) FROM memory_recall_events" + clause, + params, + ).fetchone() + metrics["recall_attempts"] = row[0] + metrics["recall_hits"] = row[1] + metrics["returned_memories"] = row[2] + if _table_exists(connection, "memory_feedback"): + clause, params = _window_clause("created_at", days, now) + rows = connection.execute( + "SELECT outcome, COUNT(*) FROM memory_feedback" + clause + + " GROUP BY outcome", + params, + ).fetchall() + outcomes = {row[0]: row[1] for row in rows} + metrics["useful"] = outcomes.get("useful", 0) + metrics["misleading"] = outcomes.get("misleading", 0) + if _table_exists(connection, "memory_index"): + clause, params = _window_clause("created_at", days, now) + prefix = " WHERE" if not clause else " AND" + metrics["new_memories"] = connection.execute( + "SELECT COUNT(*) FROM memory_index" + clause + + prefix + " source_kind='markdown'", + params, + ).fetchone()[0] + attempts = metrics["recall_attempts"] + metrics["hit_rate"] = metrics["recall_hits"] / attempts if attempts else 0.0 + metrics["unfeedback"] = max( + 0, + metrics["returned_memories"] - metrics["useful"] - metrics["misleading"], + ) + return metrics + + +def _client_metrics(connection): + clients = { + name: {field: 0 for field in CLIENT_FIELDS} + for name in ("codex", "claude", "hermes") + } + + def add(client, field, count): + family = _family(client) + clients.setdefault(family, {name: 0 for name in CLIENT_FIELDS}) + clients[family][field] += int(count or 0) + + if _table_exists(connection, "candidate_events"): + for client, count in connection.execute( + "SELECT client, COUNT(*) FROM candidate_events GROUP BY client" + ): + add(client, "candidate_events", count) + if _table_exists(connection, "memory_recall_events"): + for client, attempts, returned in connection.execute( + "SELECT client, COUNT(*), COALESCE(SUM(result_count), 0) " + "FROM memory_recall_events GROUP BY client" + ): + add(client, "recall_attempts", attempts) + add(client, "returned_memories", returned) + if _table_exists(connection, "memory_feedback"): + for client, outcome, count in connection.execute( + "SELECT client, outcome, COUNT(*) FROM memory_feedback " + "GROUP BY client, outcome" + ): + if outcome in ("useful", "misleading"): + add(client, outcome, count) + if _table_exists(connection, "memory_index"): + for raw_clients, in connection.execute( + "SELECT clients FROM memory_index WHERE source_kind='markdown'" + ): + try: + values = json.loads(raw_clients or "[]") + except (TypeError, json.JSONDecodeError): + values = [] + for client in set(values or ["unknown"]): + add(client, "new_memories", 1) + return clients + + +def _storage(config): + database_bytes = config.database_path.stat().st_size if config.database_path.is_file() else 0 + backup_bytes = 0 + if config.backup_dir.is_dir(): + backup_bytes = sum( + path.stat().st_size for path in config.backup_dir.rglob("*") if path.is_file() + ) + return {"database_bytes": database_bytes, "backup_bytes": backup_bytes} + + +def memory_value(config, now=None): + now = now or datetime.now().strftime("%Y-%m-%d %H:%M:%S") + payload = { + "all_time": _metrics_without_connection(), + "windows": { + "7d": _metrics_without_connection(), + "30d": _metrics_without_connection(), + }, + "clients": { + name: {field: 0 for field in CLIENT_FIELDS} + for name in ("codex", "claude", "hermes") + }, + "backlog": {"pending": 0, "oldest_pending_at": None}, + "storage": _storage(config), + } + if not config.database_path.is_file(): + return payload + uri = "file:{}?mode=ro".format(quote(str(config.database_path.resolve()), safe="/")) + connection = sqlite3.connect(uri, uri=True) + try: + payload["all_time"] = _metrics(connection, None, now) + payload["windows"]["7d"] = _metrics(connection, 7, now) + payload["windows"]["30d"] = _metrics(connection, 30, now) + payload["clients"] = _client_metrics(connection) + if _table_exists(connection, "candidate_events"): + row = connection.execute( + "SELECT COUNT(*), MIN(created_at) FROM candidate_events " + "WHERE status='pending'" + ).fetchone() + payload["backlog"] = { + "pending": row[0], "oldest_pending_at": row[1] + } + return payload + finally: + connection.close() + + +def _metrics_without_connection(): + return { + "recall_attempts": 0, "recall_hits": 0, "hit_rate": 0.0, + "returned_memories": 0, "useful": 0, "misleading": 0, + "unfeedback": 0, "new_memories": 0, + } diff --git a/scripts/native_auto_extract.py b/scripts/native_auto_extract.py new file mode 100644 index 0000000..da3981a --- /dev/null +++ b/scripts/native_auto_extract.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +"""Bounded automatic proposal extraction for native client session files.""" + +import hashlib +import json +import re +from pathlib import Path + +from scripts.bounded_proposals import DEFAULT_MAX_PROPOSALS +from scripts.candidate_events import CandidateEventQueue + +DEFAULT_MAX_BYTES = 64 * 1024 +DEFAULT_MAX_MESSAGES = 12 + +_USER_RULES = ( + ("preference", re.compile(r"(?:记住|以后.{0,80}(?:不要|必须|优先|默认)|默认(?:使用|采用|不要)|不要再|必须始终|优先使用|我偏好|remember|by default|always|never)")), + ("decision", re.compile(r"(?:我(?:们)?决定|决定采用|最终决定|确定使用|采用.{0,60}方案|选择.{0,60}而不是|we decided|final decision|adopt.{0,60}approach)")), + ("workflow", re.compile(r"(?:(?:以后|必须)先.{0,80}再|流程是|步骤是|first.{0,80}then)")), +) +_ASSISTANT_RULES = ( + ("fact", re.compile(r"(?:已验证[：:]?|验证结果[：:]|测试通过[：:]?|已修复[：:]?|根因[：:]|结论[：:]|verified[：:]?|tests? pass(?:ed)?[：:]?|root cause[：:])", re.IGNORECASE)), + ("workflow", re.compile(r"(?:流程是|步骤是|应当先|应该先|the workflow|steps are)")), +) +_TRANSIENT = re.compile(r"(?:现在|今天|明天|今晚|待会|下午|上午|if I am|later today)", re.IGNORECASE) +_MEMORY_ECHO = re.compile( + r"(?:共找到\s*\d+\s*条记忆|\bID:\s*\d+|内容:\s*|" + r"^(?:web )?search results? for query:)", + re.IGNORECASE, +) +_VAGUE_ASSISTANT = re.compile( + r"^(?:(?:已修复|已完成)(?:并)?(?:重新)?(?:打开|处理|完成)?(?:这|该)?(?:条)?" + r"(?:对话|任务|问题|内容)(?:串)?[。.]?|I(?:'ve| have) published and verified the live post[.]?)$", + re.IGNORECASE, +) +_META_MEMORY = re.compile(r"(?:人工审查|自动提取|误采集|记忆检索|候选(?:里|中|项)|DNA_MEMORY_PROPOSAL)", re.IGNORECASE) +_LOW_SIGNAL_STATUS = re.compile( + r"(?:publication report|same-day content|最新主题和详情|" + r"(?:全量|现有|单元)?测试(?:通过|\s*`?\d+ passed)|" + r"DNA Memory.{0,80}(?:完全可用|验证完成)|记忆总数|MCP 工具)", + re.IGNORECASE, +) +_VAGUE_REFERENCE = re.compile( + r"^(?:结论[：:]?\s*)?(?:这|它|该(?:方向|方案|系统|功能|问题))", + re.IGNORECASE, +) +_CONVERSATIONAL_STATUS = re.compile( + r"^(?:你说得对[，,]?我应该|已创建文件\s*\[)", + re.IGNORECASE, +) + + +def _text(value): + if isinstance(value, str): + return value + if isinstance(value, list): + return " ".join(part for item in value if (part := _text(item))) + if isinstance(value, dict): + for key in ("text", "content", "message", "output"): + if key in value: + text = _text(value[key]) + if text: + return text + return "" + + +def _record_message(record): + if not isinstance(record, dict): + return None + role = record.get("role") + body = record + if role not in {"user", "assistant"} and record.get("type") in {"user", "assistant"}: + role = record["type"] + if role not in {"user", "assistant"} and record.get("type") == "response_item": + body = record.get("payload") or {} + if body.get("type") != "message": + return None + role = body.get("role") + if role not in {"user", "assistant"}: + return None + content = body.get("content") + if content is None: + content = (body.get("message") or {}).get("content") + text = _text(content).strip() + return { + "role": role, + "content": text, + "project_path": record.get("cwd") or body.get("cwd"), + } if text else None + + +def _messages_from_data(data): + if isinstance(data, dict) and isinstance(data.get("messages"), list): + return [item for item in (_record_message(message) for message in data["messages"]) if item] + if isinstance(data, list): + return [item for item in (_record_message(record) for record in data) if item] + return [] + + +def _messages_from_json_tail(raw): + text = raw.decode("utf-8", errors="ignore") + decoder = json.JSONDecoder() + messages = [] + for index, character in enumerate(text): + if character != "{": + continue + try: + record, _ = decoder.raw_decode(text[index:]) + except json.JSONDecodeError: + continue + item = _record_message(record) + if item: + messages.append(item) + return messages + + +def _read_tail_bytes(path, max_bytes): + path = Path(path) + size = path.stat().st_size + with path.open("rb") as handle: + handle.seek(max(0, size - int(max_bytes))) + return handle.read(int(max_bytes)), size + + +def read_bounded_messages(path, max_bytes=DEFAULT_MAX_BYTES, max_messages=DEFAULT_MAX_MESSAGES): + """Read only the bounded tail and return user/assistant messages.""" + path = Path(path) + tail, size = _read_tail_bytes(path, max_bytes) + if path.suffix.lower() == ".json": + try: + return _messages_from_data(json.loads(tail.decode("utf-8")))[:max_messages] + except (UnicodeDecodeError, json.JSONDecodeError): + return _messages_from_json_tail(tail)[-max_messages:] + lines = tail.decode("utf-8", errors="ignore").splitlines() + if size > len(tail) and lines: + lines = lines[1:] + messages = [] + for line in lines: + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + item = _record_message(record) + if item: + messages.append(item) + return messages[-max_messages:] + + +def _sentence(text): + line = next((line.strip() for line in text.splitlines() if line.strip()), "") + parts = re.split(r"(?<=[。！？!?])|(?<=\.)\s+", line) + return next((part.strip() for part in parts if part.strip()), line) + + +def extract_automatic_proposals(messages, max_proposals=DEFAULT_MAX_PROPOSALS): + """Extract high-signal short proposals; never return the source transcript.""" + proposals = [] + seen = set() + for message in messages: + text = _sentence(message.get("content", ""))[:800].strip() + if not text: + continue + if (_MEMORY_ECHO.search(text) or _META_MEMORY.search(text) + or _LOW_SIGNAL_STATUS.search(text) or _VAGUE_REFERENCE.search(text) + or _CONVERSATIONAL_STATUS.search(text)): + continue + if message.get("role") == "user" and _TRANSIENT.search(text): + continue + if message.get("role") == "assistant" and _VAGUE_ASSISTANT.match(text): + continue + rules = _USER_RULES if message.get("role") == "user" else _ASSISTANT_RULES + for memory_type, pattern in rules: + if not pattern.search(text): + continue + key = re.sub(r"\s+", " ", text).lower() + if key in seen: + break + seen.add(key) + proposals.append({ + "type": memory_type, + "summary": text, + "confidence": "high" if message.get("role") == "user" else "medium", + "importance": 0.8 if memory_type in {"preference", "decision"} else 0.7, + }) + break + if len(proposals) >= max_proposals: + break + return proposals[:max_proposals] + + +def _session_id(path, messages): + try: + raw, size = _read_tail_bytes(path, DEFAULT_MAX_BYTES) + data = json.loads(raw.decode("utf-8")) if Path(path).suffix == ".json" and size <= len(raw) else {} + if isinstance(data, dict): + return data.get("session_id") or data.get("sessionId") or data.get("cliSessionId") + except (OSError, UnicodeError, json.JSONDecodeError): + pass + if Path(path).suffix == ".jsonl": + try: + raw, _ = _read_tail_bytes(path, DEFAULT_MAX_BYTES) + for line in raw.decode("utf-8", errors="ignore").splitlines(): + record = json.loads(line) + payload = record.get("payload") if isinstance(record, dict) else None + value = record.get("session_id") or record.get("sessionId") or record.get("cliSessionId") + if not value and isinstance(payload, dict): + value = payload.get("id") or payload.get("session_id") + if value: + return str(value) + except (OSError, json.JSONDecodeError): + pass + return Path(path).stem + + +def _project_path(path, messages): + for message in messages: + if message.get("project_path"): + return str(message["project_path"]) + return None + + +def import_native_file(path, queue, client, max_bytes=DEFAULT_MAX_BYTES): + path = Path(path).expanduser().resolve() + stat = path.stat() + source_ref = str(path) + checkpoint_ref = "native-auto:{}:{}".format(client, source_ref) + tail, _ = _read_tail_bytes(path, max_bytes) + digest = hashlib.sha256(tail).hexdigest() + checkpoint = queue.get_checkpoint(checkpoint_ref) + if checkpoint and checkpoint[2] == digest: + return {"processed": False, "enqueued": 0, "proposals": 0} + messages = read_bounded_messages(path, max_bytes=max_bytes) + proposals = extract_automatic_proposals(messages) + session_id = _session_id(path, messages) + project_path = _project_path(path, messages) + enqueued = 0 + for index, proposal in enumerate(proposals): + normalized = re.sub(r"\s+", " ", proposal["summary"]).strip().lower() + event_hash = hashlib.sha256( + (client + "|" + str(session_id) + "|" + normalized).encode() + ).hexdigest() + source_hash = hashlib.sha256( + (source_ref + "|" + digest + "|" + str(index) + "|" + normalized).encode() + ).hexdigest() + if queue.enqueue({ + "event_id": "auto_proposal_" + event_hash[:24], + "client": client, + "event_type": "memory_proposal", + "session_id": session_id, + "project_path": project_path, + "source_ref": source_ref + "#proposal=" + str(index), + "source_hash": source_hash, + "excerpt": proposal["summary"], + "memory_type": proposal["type"], + "confidence": proposal["confidence"], + "importance": proposal["importance"], + }): + enqueued += 1 + if not proposals: + pointer_hash = hashlib.sha256((source_ref + digest).encode()).hexdigest() + if queue.enqueue({ + "event_id": "auto_session_" + pointer_hash[:24], + "client": client, + "event_type": "session_updated", + "session_id": session_id, + "project_path": project_path, + "source_ref": source_ref, + "source_hash": digest, + }): + enqueued += 1 + queue.update_checkpoint(checkpoint_ref, stat.st_ino, stat.st_size, digest) + return {"processed": True, "enqueued": enqueued, "proposals": len(proposals)} diff --git a/scripts/policy.py b/scripts/policy.py new file mode 100644 index 0000000..d04a3db --- /dev/null +++ b/scripts/policy.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Deterministic storage safety and capacity checks.""" + +import re +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class PolicyResult: + allowed: bool + reason: str = "" + + +@dataclass(frozen=True) +class CapacityStatus: + state: str + size_bytes: int + writable: bool + + +RULES = ( + ("private_key", re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----")), + ("api_token", re.compile(r"\b(?:sk|ghp|xox[baprs])-[A-Za-z0-9_-]{20,}\b")), + ("password", re.compile(r"(?i)\b(?:password|passwd|pwd)\s*[:=]\s*\S+")), +) + + +def inspect_content(content: str) -> PolicyResult: + if len(content) >= 5000 and re.fullmatch(r"[A-Za-z0-9+/=\s]+", content): + return PolicyResult(False, "large_encoded_block") + for name, pattern in RULES: + if pattern.search(content): + return PolicyResult(False, name) + return PolicyResult(True) + + +def capacity_status(path: Path, warning_bytes: int, hard_bytes: int) -> CapacityStatus: + size = path.stat().st_size if path.exists() else 0 + if size >= hard_bytes: + return CapacityStatus("blocked", size, False) + if size >= warning_bytes: + return CapacityStatus("warning", size, True) + return CapacityStatus("ok", size, True) diff --git a/scripts/skill_manager.py b/scripts/skill_manager.py new file mode 100644 index 0000000..87b4828 --- /dev/null +++ b/scripts/skill_manager.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Non-destructive inventory and distribution for shared Agent Skills.""" + +import hashlib +import os +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Dict, Iterable, List + + +@dataclass(frozen=True) +class SkillFinding: + name: str + platform: str + state: str + path: str + + def to_dict(self): + return asdict(self) + + +@dataclass(frozen=True) +class SyncItem: + name: str + platform: str + action: str + source: str + target: str + + def to_dict(self): + return asdict(self) + + +def _entries(root: Path) -> Iterable[Path]: + if not root.is_dir(): + return [] + return sorted(root.iterdir(), key=lambda path: path.name.lower()) + + +def _digest(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted((item for item in root.rglob("*") if item.is_file())): + digest.update(str(path.relative_to(root)).encode()) + digest.update(path.read_bytes()) + return digest.hexdigest() + + +def _is_correct_link(target: Path, source: Path) -> bool: + return target.is_symlink() and target.exists() and target.resolve() == source.resolve() + + +def inventory(shared_root: Path, platform_roots: Dict[str, Path], registry: dict) -> List[SkillFinding]: + managed = registry.get("skills", {}) + findings = [] + for platform, root in sorted(platform_roots.items()): + for target in _entries(root): + name = target.name + source = shared_root / name + if target.is_symlink() and not target.exists(): + state = "broken_link" + elif name not in managed: + state = "platform" + elif _is_correct_link(target, source): + state = "shared" + elif source.is_dir() and target.is_dir() and _digest(source) == _digest(target): + state = "shadowed" + else: + state = "conflict" + findings.append(SkillFinding(name, platform, state, str(target))) + return findings + + +def build_sync_plan(shared_root: Path, platform_roots: Dict[str, Path], registry: dict) -> List[SyncItem]: + plan = [] + for name, settings in sorted(registry.get("skills", {}).items()): + source = shared_root / name + for platform in settings.get("targets", []): + root = platform_roots.get(platform) + target = (root / name) if root else Path("") / name + if root is None: + action = "unconfigured_platform" + elif not source.is_dir() or not (source / "SKILL.md").is_file(): + action = "missing_source" + elif _is_correct_link(target, source): + action = "ok" + elif target.is_symlink() or target.exists(): + action = "blocked_conflict" + else: + action = "create_link" + plan.append(SyncItem(name, platform, action, str(source), str(target))) + return plan + + +def apply_sync_plan(plan: List[SyncItem], apply: bool = False) -> int: + if not apply: + return 0 + changed = 0 + for item in plan: + if item.action != "create_link": + continue + target = Path(item.target) + target.parent.mkdir(parents=True, exist_ok=True) + target.symlink_to(Path(item.source), target_is_directory=True) + changed += 1 + return changed diff --git a/scripts/skills_cli.py b/scripts/skills_cli.py new file mode 100644 index 0000000..6b3212c --- /dev/null +++ b/scripts/skills_cli.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""CLI for cross-client Skill inventory, diagnosis, and safe sync.""" + +import argparse +import json +from pathlib import Path +from typing import Optional, Sequence + +from scripts.config import load_config +from scripts.skill_manager import apply_sync_plan, build_sync_plan, inventory + + +def _parser(): + parser = argparse.ArgumentParser(prog="dna skills") + parser.add_argument("--profile", type=Path) + sub = parser.add_subparsers(dest="command", required=True) + for name in ("inventory", "doctor", "sync"): + command = sub.add_parser(name) + command.add_argument("--json", action="store_true", dest="as_json") + if name == "sync": + command.add_argument("--apply", action="store_true") + return parser + + +def _registry(path: Path) -> dict: + if not path.is_file(): + return {"skills": {}} + return json.loads(path.read_text(encoding="utf-8")) + + +def _emit(payload, as_json): + if as_json: + print(json.dumps(payload, ensure_ascii=False, sort_keys=True)) + else: + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + args = _parser().parse_args(argv) + config = load_config(args.profile) + registry = _registry(config.skill_registry) + if args.command == "inventory": + items = inventory(config.skill_root, config.platform_skill_roots, registry) + _emit({"count": len(items), "items": [item.to_dict() for item in items]}, args.as_json) + return 0 + if args.command == "doctor": + items = inventory(config.skill_root, config.platform_skill_roots, registry) + issues = [item for item in items if item.state in {"broken_link", "conflict"}] + plan = build_sync_plan(config.skill_root, config.platform_skill_roots, registry) + blocked = [item for item in plan if item.action not in {"ok", "create_link"}] + _emit({ + "healthy": not issues and not blocked, + "issues": [item.to_dict() for item in issues], + "blocked": [item.to_dict() for item in blocked], + }, args.as_json) + return 0 + plan = build_sync_plan(config.skill_root, config.platform_skill_roots, registry) + changed = apply_sync_plan(plan, apply=args.apply) + _emit({ + "mode": "apply" if args.apply else "dry-run", + "changed": changed, + "items": [item.to_dict() for item in plan], + }, args.as_json) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/sync_to_claude.py b/scripts/sync_to_claude.py index f0dd9de..b107b13 100755 --- a/scripts/sync_to_claude.py +++ b/scripts/sync_to_claude.py @@ -8,18 +8,28 @@ import sqlite3 import json +import os from pathlib import Path from datetime import datetime -# DNA Memory 数据库路径 -DNA_MEMORY_DB = Path.home() / ".cc-switch/skills/dna-memory/memory/memory.db" - -# Claude Code Memory 路径 -CLAUDE_MEMORY_DIR = Path.home() / ".claude/projects/-Users-andy-Documents-AICode/memory" +# Legacy database path. Override it instead of editing this file. +DNA_MEMORY_DB = Path(os.getenv( + "DNA_MEMORY_LEGACY_DB", + Path(__file__).resolve().parents[1] / "memory" / "memory.db", +)).expanduser() + +# Claude Code memory target. Project-specific locations vary by installation. +CLAUDE_MEMORY_DIR = Path(os.getenv( + "CLAUDE_MEMORY_DIR", + Path.home() / ".claude" / "memory", +)).expanduser() CLAUDE_MEMORY_DIR.mkdir(parents=True, exist_ok=True) -# 配置文件 -CONFIG_FILE = Path.home() / ".cc-switch/skills/dna-memory/assets/config.json" +# Optional legacy configuration. Runtime configuration stays outside the repo. +CONFIG_FILE = Path(os.getenv( + "DNA_MEMORY_LEGACY_CONFIG", + Path.home() / ".config" / "dna-memory" / "legacy-config.json", +)).expanduser() def load_config(): """加载配置""" diff --git a/scripts/unified_memory.py b/scripts/unified_memory.py new file mode 100644 index 0000000..d96c7ec --- /dev/null +++ b/scripts/unified_memory.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +"""Rebuildable SQLite index for Markdown-backed long-term memories.""" + +import hashlib +import json +import sqlite3 +from pathlib import Path +from typing import Dict, Iterable, Set + + +SCHEMA_VERSION = 5 + + +class UnifiedMemoryStore: + def __init__(self, path: Path): + self.path = Path(path) + self.path.parent.mkdir(parents=True, exist_ok=True) + self.connection = sqlite3.connect(str(self.path)) + self.connection.row_factory = sqlite3.Row + self.migrate() + + def migrate(self) -> None: + self.connection.executescript(""" + CREATE TABLE IF NOT EXISTS schema_version ( + version INTEGER PRIMARY KEY, + applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + CREATE TABLE IF NOT EXISTS memory_index ( + memory_id TEXT PRIMARY KEY, + markdown_path TEXT, + type TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active', + summary TEXT NOT NULL, + content_hash TEXT NOT NULL, + source_hash TEXT, + source_refs TEXT NOT NULL DEFAULT '[]', + supersedes TEXT NOT NULL DEFAULT '[]', + superseded_by TEXT, + confidence TEXT, + importance REAL NOT NULL DEFAULT 0.5, + sensitivity TEXT NOT NULL DEFAULT 'normal', + clients TEXT NOT NULL DEFAULT '[]', + project_path TEXT, + session_id TEXT, + created_at TEXT, + updated_at TEXT, + last_recalled_at TEXT, + expires_at TEXT, + legacy_id INTEGER UNIQUE, + source_kind TEXT NOT NULL DEFAULT 'markdown' + ); + CREATE INDEX IF NOT EXISTS idx_memory_index_type ON memory_index(type); + CREATE INDEX IF NOT EXISTS idx_memory_index_source ON memory_index(source_kind); + CREATE TABLE IF NOT EXISTS memory_links ( + source_id TEXT NOT NULL, + target_id TEXT NOT NULL, + relation_type TEXT NOT NULL DEFAULT 'related', + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (source_id, target_id, relation_type), + FOREIGN KEY (source_id) REFERENCES memory_index(memory_id), + FOREIGN KEY (target_id) REFERENCES memory_index(memory_id) + ); + CREATE TABLE IF NOT EXISTS memory_feedback ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + memory_id TEXT NOT NULL, + outcome TEXT NOT NULL, + note TEXT, + client TEXT, + session_id TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (memory_id) REFERENCES memory_index(memory_id) + ); + CREATE INDEX IF NOT EXISTS idx_memory_feedback_memory + ON memory_feedback(memory_id); + CREATE TABLE IF NOT EXISTS memory_recall_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + query_hash TEXT NOT NULL, + client TEXT, + session_id TEXT, + result_count INTEGER NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + CREATE INDEX IF NOT EXISTS idx_memory_recall_events_created + ON memory_recall_events(created_at); + """) + columns = { + row[1] for row in self.connection.execute("PRAGMA table_info(memory_index)").fetchall() + } + if "source_refs" not in columns: + self.connection.execute( + "ALTER TABLE memory_index ADD COLUMN source_refs TEXT NOT NULL DEFAULT '[]'" + ) + if "recall_count" not in columns: + self.connection.execute( + "ALTER TABLE memory_index " + "ADD COLUMN recall_count INTEGER NOT NULL DEFAULT 0" + ) + if "supersedes" not in columns: + self.connection.execute( + "ALTER TABLE memory_index " + "ADD COLUMN supersedes TEXT NOT NULL DEFAULT '[]'" + ) + if "superseded_by" not in columns: + self.connection.execute( + "ALTER TABLE memory_index ADD COLUMN superseded_by TEXT" + ) + self._migrate_legacy() + self.connection.execute( + "INSERT OR IGNORE INTO schema_version(version) VALUES (?)", (SCHEMA_VERSION,) + ) + self.connection.commit() + + def _migrate_legacy(self) -> None: + exists = self.connection.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='memory'" + ).fetchone() + if not exists: + return + for row in self.connection.execute( + "SELECT id, content, type, weight, created, updated FROM memory" + ).fetchall(): + digest = hashlib.sha256(row["content"].encode()).hexdigest() + self.connection.execute(""" + INSERT OR IGNORE INTO memory_index ( + memory_id, type, summary, content_hash, importance, + created_at, updated_at, legacy_id, source_kind + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'legacy_cache') + """, ( + "legacy-{}".format(row["id"]), row["type"], row["content"], digest, + row["weight"], str(row["created"] or ""), str(row["updated"] or ""), row["id"], + )) + + def upsert_markdown(self, record: Dict[str, object]) -> bool: + existing = self.connection.execute( + "SELECT content_hash FROM memory_index WHERE memory_id=?", (record["memory_id"],) + ).fetchone() + if existing and existing["content_hash"] == record["content_hash"]: + return False + self.connection.execute(""" + INSERT INTO memory_index ( + memory_id, markdown_path, type, status, summary, content_hash, + source_hash, source_refs, supersedes, superseded_by, + confidence, importance, sensitivity, clients, + project_path, session_id, created_at, updated_at, expires_at, source_kind + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'markdown') + ON CONFLICT(memory_id) DO UPDATE SET + markdown_path=excluded.markdown_path, type=excluded.type, + status=excluded.status, summary=excluded.summary, + content_hash=excluded.content_hash, source_hash=excluded.source_hash, + source_refs=excluded.source_refs, + supersedes=excluded.supersedes, superseded_by=excluded.superseded_by, + confidence=excluded.confidence, importance=excluded.importance, + sensitivity=excluded.sensitivity, clients=excluded.clients, + project_path=excluded.project_path, session_id=excluded.session_id, + created_at=excluded.created_at, updated_at=excluded.updated_at, + expires_at=excluded.expires_at, source_kind='markdown' + """, ( + record["memory_id"], record["markdown_path"], record["type"], record["status"], + record["summary"], record["content_hash"], record.get("source_hash"), + json.dumps(record.get("source_refs", []), ensure_ascii=False), + json.dumps(record.get("supersedes", []), ensure_ascii=False), + record.get("superseded_by"), + record.get("confidence"), record.get("importance", 0.5), + record.get("sensitivity", "normal"), json.dumps(record.get("clients", []), ensure_ascii=False), + record.get("project_path"), record.get("session_id"), record.get("created_at"), + record.get("updated_at"), record.get("expires_at"), + )) + return True + + def remove_missing_markdown(self, present_ids: Set[str]) -> int: + rows = self.connection.execute( + "SELECT memory_id FROM memory_index WHERE source_kind='markdown'" + ).fetchall() + missing = [row["memory_id"] for row in rows if row["memory_id"] not in present_ids] + if missing: + self.connection.executemany( + "DELETE FROM memory_index WHERE memory_id=?", ((item,) for item in missing) + ) + return len(missing) + + def count_managed(self) -> int: + return self.connection.execute( + "SELECT COUNT(*) FROM memory_index WHERE source_kind='markdown'" + ).fetchone()[0] + + def commit(self) -> None: + self.connection.commit() + + def close(self) -> None: + self.connection.commit() + self.connection.close() diff --git a/skills/dna-memory-loop/SKILL.md b/skills/dna-memory-loop/SKILL.md new file mode 100644 index 0000000..0665641 --- /dev/null +++ b/skills/dna-memory-loop/SKILL.md @@ -0,0 +1,55 @@ +--- +name: dna-memory-loop +description: Use when a substantive task may depend on prior decisions, preferences, project state, workflows, errors, or open loops, or when a verified result should become reusable across Codex, Claude, Hermes, and an Obsidian vault. +--- + +# DNA Memory Loop + +Use DNA Memory as the cross-client long-term truth. Treat each client's native +history as a transcript source, not as a competing durable memory store. + +## Before substantive work + +1. Extract one to four distinctive terms from the request, project, error, or expected result. +2. Call `memory_recall` separately for each term. Include the real client and session ID when available. +3. Deduplicate by memory ID. Inject at most five memories and about 2,000 tokens total. +4. Use only relevant results. Current files, processes, remote state, and tests override stale memory. + +Skip recall for simple, self-contained work such as translation, current time, +one-step formatting, or a command that does not depend on history. + +## Feedback + +- Call `memory_feedback(..., outcome="useful")` only when a recalled memory changed or confirmed the work. +- Use `outcome="misleading"` when a memory sent the task in the wrong direction. +- Do not submit feedback for a result that was displayed but not used. + +## After verification + +Call `memory_remember` only for a durable preference, decision with rationale, +fact, insight, reusable workflow, proven error lesson, project state, or open +loop. Write a compact conclusion with bounded source pointers. Never store a +full transcript, credentials, raw prompts, or large tool output. + +When current evidence invalidates an older conclusion, read the older memory +IDs and pass only those exact IDs through `supersedes`. Never infer replacement +from project or type alone. + +When a native session has a stable source pointer, call `memory_close_session` +with bounded metadata. This records provenance, not transcript content. + +## Bounded proposals + +If the client cannot directly call `memory_remember`, it may append up to three +reviewable candidates to its final message: + +`DNA_MEMORY_PROPOSAL {"type":"decision|fact|workflow|preference|insight|error_lesson|project_state|open_loop","summary":"verified reusable conclusion","confidence":"high|medium|low","importance":0.0}` + +Keep each summary under 800 characters. A proposal remains a candidate until +maintenance applies type, sensitivity, capacity, and deduplication checks. + +## Degraded mode + +Memory failure must not block the main task. Continue from current evidence, +report the degraded capability once, and never fabricate recall, feedback, or +writeback success. diff --git a/skills/dna-memory-loop/agents/openai.yaml b/skills/dna-memory-loop/agents/openai.yaml new file mode 100644 index 0000000..d92c560 --- /dev/null +++ b/skills/dna-memory-loop/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "DNA Memory Loop" + short_description: "Recall before work and write back only verified conclusions." + default_prompt: "Use $dna-memory-loop to recall relevant context and write back only verified durable conclusions." diff --git a/tests/test_bounded_proposals.py b/tests/test_bounded_proposals.py new file mode 100644 index 0000000..79de552 --- /dev/null +++ b/tests/test_bounded_proposals.py @@ -0,0 +1,50 @@ +import json + +from scripts.bounded_proposals import extract_proposals + + +def test_extracts_only_explicit_bounded_proposals(): + text = """ + 普通结论不应自动入库。 + DNA_MEMORY_PROPOSAL {"type":"decision","summary":"统一记忆以 Obsidian Markdown 为真源","confidence":"high","importance":0.9} + DNA_MEMORY_PROPOSAL {"type":"fact","summary":"第二条","confidence":"medium"} + """ + + proposals = extract_proposals(text) + + assert proposals == [ + { + "type": "decision", + "summary": "统一记忆以 Obsidian Markdown 为真源", + "confidence": "high", + "importance": 0.9, + }, + {"type": "fact", "summary": "第二条", "confidence": "medium"}, + ] + + +def test_rejects_sensitive_invalid_and_unbounded_proposals(): + secret = "DNA_MEMORY_PROPOSAL " + json.dumps({ + "type": "fact", "summary": "password: hunter2" + }) + long_summary = "DNA_MEMORY_PROPOSAL " + json.dumps({ + "type": "fact", "summary": "x" * 801 + }) + invalid = "DNA_MEMORY_PROPOSAL {\"type\":\"unknown\",\"summary\":\"x\"}" + invalid_confidence = "DNA_MEMORY_PROPOSAL {\"type\":\"fact\",\"summary\":\"x\",\"confidence\":\"certain\"}" + invalid_importance = "DNA_MEMORY_PROPOSAL {\"type\":\"fact\",\"summary\":\"x\",\"importance\":2}" + + assert extract_proposals("\n".join(( + secret, long_summary, invalid, invalid_confidence, invalid_importance, + ))) == [] + + +def test_caps_proposals_and_reads_only_tail(tmp_path): + path = tmp_path / "transcript.jsonl" + path.write_text("private old content\n" + "DNA_MEMORY_PROPOSAL {\"type\":\"fact\",\"summary\":\"tail\"}\n") + + from scripts.bounded_proposals import read_tail_proposals + + assert read_tail_proposals(path, max_bytes=200, max_proposals=1) == [ + {"type": "fact", "summary": "tail"} + ] diff --git a/tests/test_bundled_skills.py b/tests/test_bundled_skills.py new file mode 100644 index 0000000..8e8261b --- /dev/null +++ b/tests/test_bundled_skills.py @@ -0,0 +1,37 @@ +from pathlib import Path + +import pytest +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +SKILLS = ("dna-memory-loop",) + + +@pytest.mark.parametrize("name", SKILLS) +def test_bundled_skill_has_cross_client_memory_contract(name): + path = ROOT / "skills" / name / "SKILL.md" + text = path.read_text(encoding="utf-8") + _, raw, body = text.split("---", 2) + meta = yaml.safe_load(raw) + + assert meta["name"] == name + assert meta["description"].startswith("Use when") + assert "Before substantive work" in body + assert "After verification" in body + assert "Memory failure must not block" in body + + +def test_memory_loop_has_bounded_active_use_gates(): + body = (ROOT / "skills/dna-memory-loop/SKILL.md").read_text(encoding="utf-8") + + assert "one to four distinctive terms" in body + assert "at most five memories" in body + assert "memory_feedback" in body + assert "memory_remember" in body + assert "Current files" in body + assert "full transcript" in body + assert "displayed but not used" in body + assert "simple, self-contained" in body + assert "supersedes" in body + assert "Never infer replacement" in body diff --git a/tests/test_candidate_events.py b/tests/test_candidate_events.py new file mode 100644 index 0000000..daa2ab2 --- /dev/null +++ b/tests/test_candidate_events.py @@ -0,0 +1,29 @@ +from scripts.candidate_events import CandidateEventQueue + + +def test_candidate_events_are_idempotent_and_secret_safe(tmp_path): + queue = CandidateEventQueue(tmp_path / "events.db") + event = {"event_id": "e1", "client": "claude", "event_type": "Stop", + "session_id": "s1", "source_ref": "/tmp/session.jsonl", + "excerpt": "password=secret123"} + assert queue.enqueue(event) is True + assert queue.enqueue(event) is False + row = queue.connection.execute( + "SELECT excerpt, source_ref FROM candidate_events WHERE event_id='e1'" + ).fetchone() + assert row[0] is None + assert row[1] == "/tmp/session.jsonl" + + +def test_candidate_events_apply_a_bound_without_blocking_proposals(tmp_path): + queue = CandidateEventQueue(tmp_path / "bounded.db", max_events=2) + assert queue.enqueue({"event_id": "e1", "client": "codex", "event_type": "task_complete"}) is True + assert queue.enqueue({"event_id": "e2", "client": "codex", "event_type": "task_complete"}) is True + assert queue.enqueue({"event_id": "e3", "client": "codex", "event_type": "task_complete"}) is False + assert queue.enqueue({ + "event_id": "proposal-1", "client": "codex", "event_type": "memory_proposal", + "memory_type": "fact", "excerpt": "bounded proposal", + }) is True + assert queue.connection.execute( + "SELECT COUNT(*) FROM candidate_events" + ).fetchone()[0] == 3 diff --git a/tests/test_claude_desktop_adapter.py b/tests/test_claude_desktop_adapter.py new file mode 100644 index 0000000..4322202 --- /dev/null +++ b/tests/test_claude_desktop_adapter.py @@ -0,0 +1,68 @@ +import json + +from scripts.candidate_events import CandidateEventQueue +from scripts.import_claude_desktop_sessions import import_sessions + + +def test_claude_desktop_import_is_metadata_only_and_idempotent(tmp_path): + sessions = tmp_path / "sessions/account/workspace" + sessions.mkdir(parents=True) + session = sessions / "local_desktop-session.json" + session.write_text(json.dumps({ + "sessionId": "desktop-session", + "cliSessionId": "cli-session", + "cwd": "/tmp/project", + "createdAt": "2026-07-10T12:00:00Z", + "lastActivityAt": "2026-07-11T12:00:00Z", + "title": "private title", + "initialMessage": "private initial message", + "systemPrompt": "private system prompt", + "account": {"email": "private-user@example.invalid"}, + "toolConfiguration": {"secret": "private tool configuration"}, + })) + (sessions / "spaces.json").write_text('{"initialMessage":"ignored"}') + nested = sessions / "local_output/.claude/tasks" + nested.mkdir(parents=True) + (nested / "1.json").write_text('{"prompt":"ignored task"}') + (sessions / "local_output/audit.jsonl").write_text("private audit content\n") + queue = CandidateEventQueue(tmp_path / "memory.db") + + first = import_sessions([tmp_path / "sessions"], queue) + second = import_sessions([tmp_path / "sessions"], queue) + + assert first == {"files": 1, "enqueued": 1, "skipped": 0} + assert second == {"files": 1, "enqueued": 0, "skipped": 1} + row = queue.connection.execute( + "SELECT client,event_type,session_id,project_path,source_ref,source_hash,excerpt " + "FROM candidate_events" + ).fetchone() + assert tuple(row[:5]) == ( + "claude-desktop", "session_updated", "desktop-session", "/tmp/project", + str(session.resolve()), + ) + assert len(row[5]) == 64 + assert row[6] is None + serialized = json.dumps(tuple(row)) + for private in ( + "private title", "private initial message", "private system prompt", + "private-user@example.invalid", "private tool configuration", + "private audit content", "ignored task", + ): + assert private not in serialized + + +def test_claude_desktop_import_uses_cli_session_id_and_skips_invalid_json(tmp_path): + root = tmp_path / "sessions" + root.mkdir() + (root / "local_bad.json").write_text("not-json") + valid = root / "local_cli.json" + valid.write_text(json.dumps({"cliSessionId": "cli-only", "cwd": None})) + queue = CandidateEventQueue(tmp_path / "memory.db") + + result = import_sessions([root], queue) + + assert result == {"files": 2, "enqueued": 1, "skipped": 1} + row = queue.connection.execute( + "SELECT session_id,project_path FROM candidate_events" + ).fetchone() + assert tuple(row) == ("cli-only", None) diff --git a/tests/test_claude_desktop_config.py b/tests/test_claude_desktop_config.py new file mode 100644 index 0000000..0e8234d --- /dev/null +++ b/tests/test_claude_desktop_config.py @@ -0,0 +1,103 @@ +import json + +import pytest + +from scripts.configure_claude_desktop import configure, rollback + + +def paths(tmp_path): + config = tmp_path / "Claude-3p/claude_desktop_config.json" + config.parent.mkdir() + legacy = tmp_path / "memory.jsonl" + legacy.write_text('{"legacy":"keep me"}\n') + return config, legacy, tmp_path / "backups" + + +def desired(tmp_path): + return { + "python": str(tmp_path / "venv/bin/python"), + "server": str(tmp_path / "memory_mcp.py"), + "profile": str(tmp_path / "profile.json"), + } + + +def test_dry_run_reports_replacement_without_writes(tmp_path): + config, legacy, backup_dir = paths(tmp_path) + original = json.dumps({ + "mcpServers": { + "dna-memory": {"name": "dna-memory", "command": "mcp-server-memory"}, + "other": {"command": "other-server"}, + } + }).encode() + config.write_bytes(original) + + result = configure( + config, legacy_memory=legacy, backup_dir=backup_dir, apply=False, + **desired(tmp_path) + ) + + assert result["status"] == "would_replace" + assert config.read_bytes() == original + assert not backup_dir.exists() + + +def test_apply_backs_up_replaces_only_target_and_rollback_is_byte_identical(tmp_path): + config, legacy, backup_dir = paths(tmp_path) + original = b'{\n "mcpServers": {\n "dna-memory": {"command": "mcp-server-memory"},\n "other": {"command": "other-server"}\n }\n}\n' + config.write_bytes(original) + legacy_original = legacy.read_bytes() + + result = configure( + config, legacy_memory=legacy, backup_dir=backup_dir, apply=True, + stamp="20260712T010203", **desired(tmp_path) + ) + + assert result["status"] == "replaced" + backup = result["config_backup"] + assert backup.endswith("claude_desktop_config.json.20260712T010203.bak") + assert (backup_dir / "claude_desktop_config.json.20260712T010203.bak").read_bytes() == original + assert (backup_dir / "memory.jsonl.20260712T010203.bak").read_bytes() == legacy_original + assert legacy.read_bytes() == legacy_original + payload = json.loads(config.read_text()) + assert payload["mcpServers"]["other"] == {"command": "other-server"} + assert payload["mcpServers"]["dna-memory"] == { + "command": desired(tmp_path)["python"], + "args": [desired(tmp_path)["server"]], + "env": {"DNA_MEMORY_PROFILE": desired(tmp_path)["profile"]}, + } + + restored = rollback(config, backup) + + assert restored["status"] == "rolled_back" + assert config.read_bytes() == original + + +def test_custom_target_requires_explicit_replace_flag(tmp_path): + config, legacy, backup_dir = paths(tmp_path) + config.write_text(json.dumps({ + "mcpServers": {"dna-memory": {"command": "/custom/server"}} + })) + + with pytest.raises(ValueError, match="custom dna-memory server"): + configure( + config, legacy_memory=legacy, backup_dir=backup_dir, apply=True, + **desired(tmp_path) + ) + + assert not backup_dir.exists() + + +@pytest.mark.parametrize("raw", ["not-json", '{"other": {}}']) +def test_invalid_or_missing_mcp_servers_fails_without_writes(tmp_path, raw): + config, legacy, backup_dir = paths(tmp_path) + config.write_text(raw) + original = config.read_bytes() + + with pytest.raises(ValueError): + configure( + config, legacy_memory=legacy, backup_dir=backup_dir, apply=True, + **desired(tmp_path) + ) + + assert config.read_bytes() == original + assert not backup_dir.exists() diff --git a/tests/test_claudian_adapter.py b/tests/test_claudian_adapter.py new file mode 100644 index 0000000..e61eecf --- /dev/null +++ b/tests/test_claudian_adapter.py @@ -0,0 +1,32 @@ +import json + +from scripts.candidate_events import CandidateEventQueue +from scripts.import_claudian_sessions import import_sessions + + +def test_claudian_import_is_metadata_only_and_idempotent(tmp_path): + sessions = tmp_path / "sessions" + sessions.mkdir() + session = sessions / "conv-1.meta.json" + session.write_text(json.dumps({ + "id": "conv-1", "providerId": "claude", "sessionId": "provider-1", + "updatedAt": 1234, "title": "private title must not be copied", + "usage": {"inputTokens": 999}, + "providerState": {"subagentData": {"x": {"toolOutput": "secret output"}}}, + })) + queue = CandidateEventQueue(tmp_path / "memory.db") + + first = import_sessions([sessions], queue, project_path=tmp_path) + second = import_sessions([sessions], queue, project_path=tmp_path) + + assert first == {"files": 1, "enqueued": 1, "skipped": 0} + assert second == {"files": 1, "enqueued": 0, "skipped": 1} + row = queue.connection.execute( + "SELECT client,event_type,session_id,project_path,source_ref,excerpt FROM candidate_events" + ).fetchone() + assert tuple(row) == ( + "claudian", "session_updated", "conv-1", str(tmp_path), str(session.resolve()), None, + ) + serialized = json.dumps(tuple(row)) + assert "private title" not in serialized + assert "secret output" not in serialized diff --git a/tests/test_client_adapters.py b/tests/test_client_adapters.py new file mode 100644 index 0000000..824fae9 --- /dev/null +++ b/tests/test_client_adapters.py @@ -0,0 +1,231 @@ +import json +from pathlib import Path +import subprocess +import sys + +from scripts.candidate_events import CandidateEventQueue + + +def test_claude_hook_keeps_only_lifecycle_metadata(tmp_path): + from scripts.client_event_hook import capture_payload + + database = tmp_path / "memory.db" + payload = { + "session_id": "claude-session", + "cwd": "/tmp/project", + "hook_event_name": "Stop", + "transcript_path": "/tmp/transcript.jsonl", + "prompt": "private full prompt must not be copied", + "tool_output": "large output must not be copied", + } + + assert capture_payload(payload, database) is True + queue = CandidateEventQueue(database) + row = queue.connection.execute( + "SELECT client, event_type, session_id, project_path, source_ref, excerpt " + "FROM candidate_events" + ).fetchone() + assert tuple(row) == ( + "claude", "Stop", "claude-session", "/tmp/project", + "/tmp/transcript.jsonl", None, + ) + + +def test_claude_hook_never_blocks_client_on_invalid_input(tmp_path): + script = Path(__file__).parents[1] / "scripts/client_event_hook.py" + result = subprocess.run( + [sys.executable, str(script)], input="not-json", cwd=str(tmp_path), + text=True, capture_output=True, + ) + assert result.returncode == 0 + assert result.stdout == "" + + +def test_claude_hook_reads_bounded_tail_for_explicit_proposal(tmp_path): + from scripts.client_event_hook import capture_payload + + transcript = tmp_path / "transcript.jsonl" + transcript.write_text( + "private old content\n" + "DNA_MEMORY_PROPOSAL {\"type\":\"workflow\",\"summary\":\"只保存验证后的结论\"}\n" + ) + database = tmp_path / "memory.db" + + assert capture_payload({ + "session_id": "claude-session", + "cwd": "/tmp/project", + "hook_event_name": "Stop", + "transcript_path": str(transcript), + }, database) is True + row = CandidateEventQueue(database).connection.execute( + "SELECT event_type,memory_type,excerpt FROM candidate_events " + "WHERE event_type='memory_proposal'" + ).fetchone() + assert tuple(row) == ("memory_proposal", "workflow", "只保存验证后的结论") + + +def test_claude_hook_prefers_last_assistant_message_and_ignores_prompt(tmp_path): + from scripts.client_event_hook import capture_payload + + database = tmp_path / "memory.db" + assert capture_payload({ + "session_id": "claude-session", + "hook_event_name": "Stop", + "prompt": "DNA_MEMORY_PROPOSAL {\"type\":\"fact\",\"summary\":\"用户注入\"}", + "last_assistant_message": "DNA_MEMORY_PROPOSAL {\"type\":\"fact\",\"summary\":\"Claude 结论\"}", + }, database) is True + + rows = CandidateEventQueue(database).connection.execute( + "SELECT excerpt FROM candidate_events WHERE event_type='memory_proposal'" + ).fetchall() + assert [row[0] for row in rows] == ["Claude 结论"] + + +def test_codex_importer_reads_only_appended_bytes_and_checkpoints(tmp_path): + from scripts.import_codex_rollouts import import_rollout + + rollout = tmp_path / "rollout-2026-01-01T00-00-00-session-123.jsonl" + first_records = [ + {"type": "session_meta", "payload": { + "id": "session-123", "cwd": "/tmp/project", + "base_instructions": "must not be copied", + }}, + {"type": "event_msg", "payload": { + "type": "user_message", "message": "private user message", + "image": "data:image/png;base64," + ("A" * 10000), + }}, + {"type": "response_item", "payload": { + "type": "function_call_output", "output": "huge tool output", + }}, + ] + rollout.write_text("".join(json.dumps(item) + "\n" for item in first_records)) + queue = CandidateEventQueue(tmp_path / "memory.db") + + first = import_rollout(rollout, queue) + second = import_rollout(rollout, queue) + with rollout.open("a") as handle: + handle.write(json.dumps({ + "type": "turn_context", + "payload": {"turn_id": "turn-2", "cwd": "/tmp/project"}, + }) + "\n") + handle.write(json.dumps({ + "type": "event_msg", "payload": {"type": "task_complete"}, + }) + "\n") + third = import_rollout(rollout, queue) + + assert first == {"processed": 3, "enqueued": 1} + assert second == {"processed": 0, "enqueued": 0} + assert third == {"processed": 2, "enqueued": 0} + checkpoint = queue.connection.execute( + "SELECT inode, offset, source_hash FROM import_checkpoints WHERE source_ref=?", + (str(rollout.resolve()),), + ).fetchone() + assert checkpoint[0] == rollout.stat().st_ino + assert checkpoint[1] == rollout.stat().st_size + assert len(checkpoint[2]) == 64 + rows = queue.connection.execute( + "SELECT source_ref, excerpt FROM candidate_events ORDER BY created_at, event_id" + ).fetchall() + assert len(rows) == 1 + assert all("#byte=" in row[0] and row[1] is None for row in rows) + serialized = json.dumps([tuple(row) for row in rows]) + assert "private user message" not in serialized + assert "base64" not in serialized + assert "huge tool output" not in serialized + + +def test_codex_checkpoint_stays_stable_when_small_file_grows(tmp_path): + from scripts.import_codex_rollouts import import_rollout + + rollout = tmp_path / "small.jsonl" + rollout.write_text(json.dumps({ + "type": "session_meta", "payload": {"id": "small-session"}, + }) + "\n") + queue = CandidateEventQueue(tmp_path / "memory.db") + assert import_rollout(rollout, queue)["processed"] == 1 + + with rollout.open("a") as handle: + handle.write(json.dumps({ + "type": "event_msg", "payload": {"type": "task_complete"}, + }) + "\n") + + assert import_rollout(rollout, queue) == {"processed": 1, "enqueued": 0} + + +def test_codex_imports_only_explicit_proposal_marker(tmp_path): + from scripts.import_codex_rollouts import import_rollout + + rollout = tmp_path / "rollout-session-123.jsonl" + rollout.write_text(json.dumps({ + "type": "response_item", + "payload": { + "type": "message", + "content": "普通正文 DNA_MEMORY_PROPOSAL {\"type\":\"fact\",\"summary\":\"持久化规则\"}", + }, + }) + "\n") + queue = CandidateEventQueue(tmp_path / "memory.db") + + assert import_rollout(rollout, queue) == {"processed": 1, "enqueued": 1} + row = queue.connection.execute( + "SELECT event_type,memory_type,excerpt FROM candidate_events" + ).fetchone() + assert tuple(row) == ("memory_proposal", "fact", "持久化规则") + + +def test_codex_backfills_tail_proposal_after_legacy_checkpoint(tmp_path): + from scripts.import_codex_rollouts import _fingerprint, import_rollout + + rollout = tmp_path / "legacy.jsonl" + rollout.write_text(json.dumps({ + "type": "response_item", + "payload": {"content": "DNA_MEMORY_PROPOSAL {\"type\":\"fact\",\"summary\":\"旧会话尾部\"}"}, + }) + "\n") + queue = CandidateEventQueue(tmp_path / "memory.db") + queue.update_checkpoint( + str(rollout.resolve()), rollout.stat().st_ino, + rollout.stat().st_size, _fingerprint(rollout), + ) + + assert import_rollout(rollout, queue) == {"processed": 0, "enqueued": 1} + assert import_rollout(rollout, queue) == {"processed": 0, "enqueued": 0} + + +def test_codex_accepts_assistant_output_text_and_ignores_user_marker(tmp_path): + from scripts.import_codex_rollouts import import_rollout + + rollout = tmp_path / "roles.jsonl" + records = [ + {"type": "response_item", "payload": { + "type": "message", "role": "user", + "content": [{"type": "input_text", "text": "DNA_MEMORY_PROPOSAL {\"type\":\"fact\",\"summary\":\"用户注入\"}"}], + }}, + {"type": "response_item", "payload": { + "type": "message", "role": "assistant", + "content": [{"type": "output_text", "text": "DNA_MEMORY_PROPOSAL {\"type\":\"fact\",\"summary\":\"助手结论\"}"}], + }}, + ] + rollout.write_text("".join(json.dumps(record, ensure_ascii=False) + "\n" for record in records)) + queue = CandidateEventQueue(tmp_path / "memory.db") + + assert import_rollout(rollout, queue)["enqueued"] == 1 + assert queue.connection.execute( + "SELECT excerpt FROM candidate_events WHERE event_type='memory_proposal'" + ).fetchone()[0] == "助手结论" + + +def test_codex_caps_proposals_across_the_whole_tail(tmp_path): + from scripts.import_codex_rollouts import import_rollout + + rollout = tmp_path / "cap.jsonl" + records = [{"type": "response_item", "payload": { + "type": "message", "role": "assistant", + "content": [{"type": "output_text", "text": + "DNA_MEMORY_PROPOSAL {\"type\":\"fact\",\"summary\":\"结论%s\"}" % index}], + }} for index in range(4)] + rollout.write_text("".join(json.dumps(record, ensure_ascii=False) + "\n" for record in records)) + queue = CandidateEventQueue(tmp_path / "memory.db") + + import_rollout(rollout, queue) + assert queue.connection.execute( + "SELECT COUNT(*) FROM candidate_events WHERE event_type='memory_proposal'" + ).fetchone()[0] == 3 diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..531ef71 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,55 @@ +import json +from pathlib import Path + +import scripts.config as config_module +from scripts.config import load_config + + +def test_profile_overrides_defaults_and_expands_paths(tmp_path): + profile = tmp_path / "profile.json" + profile.write_text(json.dumps({ + "knowledge_root": "~/Documents/Memory-Vault", + "skill_root": "~/.agents/skills", + "database_path": "~/dna/memory.db", + "claude_desktop_session_dirs": ["~/Claude Sessions"], + })) + + config = load_config(profile) + + assert config.knowledge_root == Path.home() / "Documents/Memory-Vault" + assert config.skill_root == Path.home() / ".agents/skills" + assert config.database_path == Path.home() / "dna/memory.db" + assert config.claude_desktop_session_dirs == (Path.home() / "Claude Sessions",) + + +def test_environment_profile_is_supported(tmp_path, monkeypatch): + profile = tmp_path / "profile.json" + profile.write_text(json.dumps({"knowledge_root": str(tmp_path / "vault")})) + monkeypatch.setenv("DNA_MEMORY_PROFILE", str(profile)) + + assert load_config().knowledge_root == tmp_path / "vault" + + +def test_default_profile_is_loaded_when_present(tmp_path, monkeypatch): + profile = tmp_path / "profile.json" + profile.write_text(json.dumps({"knowledge_root": str(tmp_path / "default-vault")})) + monkeypatch.delenv("DNA_MEMORY_PROFILE", raising=False) + monkeypatch.setattr(config_module, "DEFAULT_PROFILE", profile) + + assert load_config().knowledge_root == tmp_path / "default-vault" + + +def test_claude_desktop_session_dirs_default_to_empty_tuple(tmp_path): + profile = tmp_path / "profile.json" + profile.write_text("{}") + + assert load_config(profile).claude_desktop_session_dirs == () + + +def test_hermes_state_db_defaults_to_none_and_expands_path(tmp_path): + profile = tmp_path / "profile.json" + profile.write_text(json.dumps({"hermes_state_db": "~/hermes/state.db"})) + + config = load_config(profile) + + assert config.hermes_state_db == Path.home() / "hermes/state.db" diff --git a/tests/test_dna_dispatch.py b/tests/test_dna_dispatch.py new file mode 100644 index 0000000..d25e3bb --- /dev/null +++ b/tests/test_dna_dispatch.py @@ -0,0 +1,35 @@ +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_convenience_entry_has_no_install_path_hardcoding(): + text = (ROOT / "dna").read_text(encoding="utf-8") + assert ".cc-switch/skills/dna-memory" not in text + + +def test_old_command_help_runs_outside_repository(tmp_path): + result = subprocess.run( + [sys.executable, str(ROOT / "dna.py"), "manage", "--help"], + cwd=str(tmp_path), capture_output=True, text=True, + ) + assert result.returncode == 0, result.stderr + + +def test_new_command_help_runs_outside_repository(tmp_path): + result = subprocess.run( + [sys.executable, str(ROOT / "dna.py"), "memory", "--help"], + cwd=str(tmp_path), capture_output=True, text=True, + ) + assert result.returncode == 0, result.stderr + + +def test_convenience_entry_forwards_new_namespaces(tmp_path): + result = subprocess.run( + [sys.executable, str(ROOT / "dna"), "skills", "--help"], + cwd=str(tmp_path), capture_output=True, text=True, + ) + assert result.returncode == 0, result.stderr diff --git a/tests/test_hermes_adapter.py b/tests/test_hermes_adapter.py new file mode 100644 index 0000000..8162d42 --- /dev/null +++ b/tests/test_hermes_adapter.py @@ -0,0 +1,150 @@ +import hashlib +import json +import os +import sqlite3 + +import pytest + +from scripts.candidate_events import CandidateEventQueue +from scripts.import_hermes_sessions import import_sessions + + +def source_db(path): + connection = sqlite3.connect(str(path)) + connection.executescript(""" + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, source TEXT NOT NULL, started_at REAL NOT NULL, + ended_at REAL, message_count INTEGER DEFAULT 0, cwd TEXT + ); + CREATE TABLE messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, session_id TEXT NOT NULL, + role TEXT NOT NULL, content TEXT, tool_calls TEXT, timestamp REAL NOT NULL + ); + """) + connection.execute( + "INSERT INTO sessions VALUES ('s1', 'desktop', 1, NULL, 2, '/tmp/hermes')" + ) + connection.executemany( + "INSERT INTO messages(session_id,role,content,tool_calls,timestamp) " + "VALUES ('s1', ?, ?, ?, ?)", + [("user", "private message one", "private tool one", 1), + ("assistant", "private message two", "private tool two", 2)], + ) + connection.commit() + connection.close() + + +def test_hermes_import_is_read_only_incremental_and_idempotent(tmp_path): + source = tmp_path / "state.db" + source_db(source) + original = hashlib.sha256(source.read_bytes()).hexdigest() + queue = CandidateEventQueue(tmp_path / "memory.db") + + first = import_sessions(source, queue) + second = import_sessions(source, queue) + + assert first == {"sessions": 1, "enqueued": 1, "skipped": 0} + assert second == {"sessions": 1, "enqueued": 0, "skipped": 1} + row = queue.connection.execute( + "SELECT client,event_type,session_id,project_path,source_ref,excerpt " + "FROM candidate_events" + ).fetchone() + assert tuple(row) == ( + "hermes", "session_updated", "s1", "/tmp/hermes", + str(source.resolve()) + "#session=s1&max_message_id=2", None, + ) + serialized = json.dumps(tuple(row)) + assert "private message" not in serialized + assert "private tool" not in serialized + assert hashlib.sha256(source.read_bytes()).hexdigest() == original + + connection = sqlite3.connect(str(source)) + connection.execute( + "INSERT INTO messages(session_id,role,content,tool_calls,timestamp) " + "VALUES ('s1', 'user', 'private message three', 'private tool three', 3)" + ) + connection.commit() + connection.close() + + third = import_sessions(source, queue) + assert third == {"sessions": 1, "enqueued": 1, "skipped": 0} + assert queue.connection.execute( + "SELECT COUNT(*) FROM candidate_events" + ).fetchone()[0] == 2 + assert queue.connection.execute( + "SELECT MAX(source_ref) FROM candidate_events" + ).fetchone()[0].endswith("max_message_id=3") + + +def test_hermes_import_rejects_incompatible_schema_without_checkpoint(tmp_path): + source = tmp_path / "bad.db" + connection = sqlite3.connect(str(source)) + connection.execute("CREATE TABLE sessions (id TEXT PRIMARY KEY)") + connection.commit() + connection.close() + queue = CandidateEventQueue(tmp_path / "memory.db") + + with pytest.raises(ValueError, match="required Hermes schema"): + import_sessions(source, queue) + + assert queue.connection.execute( + "SELECT COUNT(*) FROM import_checkpoints" + ).fetchone()[0] == 0 + + +def test_hermes_imports_bounded_assistant_proposal_only(tmp_path): + source = tmp_path / "state.db" + source_db(source) + connection = sqlite3.connect(str(source)) + connection.execute( + "INSERT INTO messages(session_id,role,content,tool_calls,timestamp) VALUES (?, ?, ?, ?, ?)", + ("s1", "assistant", "DNA_MEMORY_PROPOSAL {\"type\":\"decision\",\"summary\":\"Hermes 只读接入\"}", None, 3), + ) + connection.commit() + connection.close() + queue = CandidateEventQueue(tmp_path / "memory.db") + + assert import_sessions(source, queue)["enqueued"] == 2 + row = queue.connection.execute( + "SELECT event_type,memory_type,excerpt FROM candidate_events WHERE event_type='memory_proposal'" + ).fetchone() + assert tuple(row) == ("memory_proposal", "decision", "Hermes 只读接入") + + +def test_hermes_backfills_proposal_when_legacy_checkpoint_is_current(tmp_path): + source = tmp_path / "state.db" + source_db(source) + connection = sqlite3.connect(str(source)) + connection.execute( + "UPDATE messages SET content=? WHERE id=2", + ("DNA_MEMORY_PROPOSAL {\"type\":\"fact\",\"summary\":\"旧 Hermes 会话\"}",), + ) + connection.commit() + connection.close() + queue = CandidateEventQueue(tmp_path / "memory.db") + + assert import_sessions(source, queue)["enqueued"] == 2 + queue.connection.execute("DELETE FROM candidate_events WHERE event_type='memory_proposal'") + queue.connection.commit() + + assert import_sessions(source, queue)["enqueued"] == 1 + assert import_sessions(source, queue)["enqueued"] == 0 + + +def test_hermes_caps_proposals_across_recent_messages(tmp_path): + source = tmp_path / "state.db" + source_db(source) + connection = sqlite3.connect(str(source)) + for index in range(4): + connection.execute( + "INSERT INTO messages(session_id,role,content,tool_calls,timestamp) VALUES (?, ?, ?, ?, ?)", + ("s1", "assistant", "DNA_MEMORY_PROPOSAL {\"type\":\"fact\",\"summary\":\"结论%s\"}" % index, None, 3 + index), + ) + connection.commit() + connection.close() + queue = CandidateEventQueue(tmp_path / "memory.db") + + import_sessions(source, queue) + assert queue.connection.execute( + "SELECT COUNT(*) FROM candidate_events WHERE event_type='memory_proposal'" + ).fetchone()[0] == 3 diff --git a/tests/test_markdown_memory.py b/tests/test_markdown_memory.py new file mode 100644 index 0000000..f390ff0 --- /dev/null +++ b/tests/test_markdown_memory.py @@ -0,0 +1,87 @@ +from scripts.markdown_memory import reindex_markdown +from scripts.unified_memory import UnifiedMemoryStore + + +def write_page(root, name="decision.md", memory_id="mem_1", summary="统一结论"): + page = root / name + page.write_text(f"""--- +id: {memory_id} +type: decision +status: active +confidence: high +importance: 0.88 +clients: + - codex +tags: + - memory + - decision +--- + +{summary} + +这是证据说明。 +""") + return page + + +def test_reindex_is_idempotent_and_removes_deleted_pages(tmp_path): + root = tmp_path / "vault" + root.mkdir() + page = write_page(root) + store = UnifiedMemoryStore(tmp_path / "memory.db") + + first = reindex_markdown(root, store) + second = reindex_markdown(root, store) + assert first.indexed == 1 + assert second.indexed == 0 + assert store.count_managed() == 1 + + page.unlink() + result = reindex_markdown(root, store) + assert result.removed == 1 + assert store.count_managed() == 0 + store.close() + + +def test_reindex_skips_unmanaged_markdown(tmp_path): + root = tmp_path / "vault" + root.mkdir() + (root / "ordinary.md").write_text("# 普通笔记") + store = UnifiedMemoryStore(tmp_path / "memory.db") + + result = reindex_markdown(root, store) + + assert result.scanned == 1 + assert result.skipped == 1 + assert store.count_managed() == 0 + store.close() + + +def test_reindex_restores_supersede_relationships_from_markdown(tmp_path): + root = tmp_path / "vault" + root.mkdir() + (root / "current.md").write_text("""--- +id: mem_current +type: project_state +status: active +confidence: high +importance: 0.9 +supersedes: + - mem_old +superseded_by: null +tags: + - memory + - project_state +--- + +当前结论 +""") + store = UnifiedMemoryStore(tmp_path / "memory.db") + + reindex_markdown(root, store) + + row = store.connection.execute( + "SELECT supersedes, superseded_by FROM memory_index WHERE memory_id='mem_current'" + ).fetchone() + assert tuple(row) == ('["mem_old"]', None) + store.close() diff --git a/tests/test_memory_cli.py b/tests/test_memory_cli.py new file mode 100644 index 0000000..ad6487d --- /dev/null +++ b/tests/test_memory_cli.py @@ -0,0 +1,211 @@ +import json +import sqlite3 + +from scripts.candidate_events import CandidateEventQueue +from scripts.config import load_config +from scripts.memory_cli import main +from scripts.unified_memory import UnifiedMemoryStore + + +def profile_file(tmp_path, **overrides): + vault = tmp_path / "vault" + vault.mkdir() + profile = tmp_path / "profile.json" + values = { + "knowledge_root": str(vault), + "database_path": str(tmp_path / "memory.db"), + "skill_root": str(tmp_path / "shared"), + "skill_registry": str(tmp_path / "skills.json"), + } + values.update(overrides) + profile.write_text(json.dumps(values)) + return profile, vault + + +def seed_cli_value_data(config): + store = UnifiedMemoryStore(config.database_path) + store.connection.execute( + "INSERT INTO memory_recall_events " + "(query_hash,client,result_count,created_at) VALUES (?, 'codex', 1, ?)", + ("a" * 64, "2026-07-10 12:00:00"), + ) + store.connection.commit() + store.close() + queue = CandidateEventQueue(config.database_path) + queue.enqueue({ + "event_id": "h1", "client": "hermes", "event_type": "session_updated" + }) + queue.connection.close() + + +def test_memory_status_outputs_json(tmp_path, capsys): + profile, _ = profile_file(tmp_path) + + assert main(["--profile", str(profile), "status", "--json"]) == 0 + payload = json.loads(capsys.readouterr().out) + + assert payload["truth_root_exists"] is True + assert payload["capacity"]["state"] == "ok" + assert payload["managed_records"] == 0 + + +def test_memory_reindex_reports_counts(tmp_path, capsys): + profile, vault = profile_file(tmp_path) + (vault / "note.md").write_text("# ordinary") + + assert main(["--profile", str(profile), "reindex", "--json"]) == 0 + payload = json.loads(capsys.readouterr().out) + + assert payload == {"indexed": 0, "removed": 0, "scanned": 1, "skipped": 1} + + +def test_memory_reindex_stops_at_database_hard_limit(tmp_path, capsys): + database = tmp_path / "memory.db" + database.write_bytes(b"already too large") + profile, _ = profile_file(tmp_path, hard_bytes=1, warning_bytes=0) + + assert main(["--profile", str(profile), "reindex", "--json"]) == 2 + payload = json.loads(capsys.readouterr().out) + + assert payload["error"] == "capacity_blocked" + + +def test_record_limit_is_warning_not_write_block(tmp_path, capsys): + profile, _ = profile_file(tmp_path, max_records=0) + + assert main(["--profile", str(profile), "status", "--json"]) == 0 + payload = json.loads(capsys.readouterr().out) + + assert payload["capacity"]["state"] == "warning" + assert payload["capacity"]["writable"] is True + + +def test_memory_maintain_daily_outputs_bounded_json(tmp_path, capsys): + profile, _ = profile_file(tmp_path) + + assert main([ + "--profile", str(profile), "maintain", "daily", "--json", + "--now", "2026-07-11 12:00:00", + ]) == 0 + payload = json.loads(capsys.readouterr().out) + + assert payload == { + "compacted": 0, "crystallized": 0, "deleted": 0, + "expired": 0, "rejected": 0, + } + + +def test_memory_value_outputs_bounded_json(tmp_path, capsys): + profile, _ = profile_file(tmp_path, backup_dir=str(tmp_path / "backups")) + seed_cli_value_data(load_config(profile)) + + assert main([ + "--profile", str(profile), "value", "--json", + "--now", "2026-07-11 12:00:00", + ]) == 0 + payload = json.loads(capsys.readouterr().out) + + assert payload["all_time"]["recall_attempts"] == 1 + assert payload["clients"]["hermes"]["candidate_events"] == 1 + assert "private durable summary" not in json.dumps(payload) + + +def test_memory_coverage_distinguishes_native_and_explicit_surfaces( + tmp_path, capsys, monkeypatch): + hermes_db = tmp_path / "state.db" + connection = sqlite3.connect(hermes_db) + connection.execute( + "CREATE TABLE sessions " + "(id TEXT PRIMARY KEY, source TEXT NOT NULL, started_at REAL NOT NULL)" + ) + connection.executemany( + "INSERT INTO sessions VALUES (?, ?, 1)", + (("h-cli", "cli"), ("h-tui", "tui"), ("h-feishu", "feishu")), + ) + connection.commit() + connection.close() + profile, _ = profile_file(tmp_path, hermes_state_db=str(hermes_db)) + codex = tmp_path / ".codex" / "sessions" + codex.mkdir(parents=True) + codex_desktop = codex / "desktop.jsonl" + codex_desktop.write_text(json.dumps({ + "type": "session_meta", + "payload": { + "originator": "Codex Desktop", + "source": {"subagent": {"thread_spawn": {"parent_thread_id": "private"}}}, + }, + }) + "\n") + codex_cli = codex / "cli.jsonl" + codex_cli.write_text(json.dumps({ + "type": "session_meta", + "payload": {"originator": "codex-tui", "source": "cli"}, + }) + "\n") + claude = tmp_path / ".claude" / "projects" + claude.mkdir(parents=True) + claude_desktop = claude / "desktop.jsonl" + claude_desktop.write_text(json.dumps({"entrypoint": "claude-desktop-3p"}) + "\n") + claude_cli = claude / "cli.jsonl" + claude_cli.write_text(json.dumps({"entrypoint": "sdk-cli"}) + "\n") + claude_sdk = claude / "sdk.jsonl" + claude_sdk.write_text(json.dumps({"entrypoint": "sdk-ts"}) + "\n") + from scripts.candidate_events import CandidateEventQueue + from scripts.import_native_history import SourceSpec + from scripts import client_coverage + + queue = CandidateEventQueue(tmp_path / "memory.db") + for client, session in ( + ("codex", codex_desktop), ("codex", codex_cli), + ("claude-code", claude_desktop), ("claude-code", claude_cli), + ("claude-code", claude_sdk), + ): + stat = session.stat() + queue.update_checkpoint( + "native-auto:{}:{}".format(client, session.resolve()), + stat.st_ino, stat.st_size, "digest", + ) + for session_id in ("h-cli", "h-tui", "h-feishu"): + queue.update_checkpoint( + "hermes:{}#{}".format(hermes_db.resolve(), session_id), 0, 0, "digest" + ) + queue.connection.close() + monkeypatch.setattr(client_coverage, "configured_paths", lambda config: { + "codex": SourceSpec((codex,), ("*.jsonl",)), + "claude-code": SourceSpec((claude,), ("*.jsonl",)), + }) + monkeypatch.setattr(client_coverage, "_launch_agent", lambda home, label: { + "installed": True, "loaded": True, "last_exit_code": 0, + }) + + assert main(["--profile", str(profile), "coverage", "--json"]) == 0 + payload = json.loads(capsys.readouterr().out) + + assert payload["surfaces"]["codex-desktop"]["entry_evidence"]["verified"] is True + assert payload["surfaces"]["codex-desktop"]["entry_evidence"]["matched_files"] == 1 + assert payload["surfaces"]["codex-desktop"]["entry_evidence"]["markers"] == { + "Codex Desktop/subagent": 1, + } + assert payload["surfaces"]["codex-desktop"]["source"]["checkpointed_files"] == 1 + assert payload["surfaces"]["codex-cli"]["entry_evidence"]["verified"] is True + assert payload["surfaces"]["codex-cli"]["entry_evidence"]["matched_files"] == 1 + assert payload["surfaces"]["codex-cli"]["source"]["checkpointed_files"] == 1 + assert payload["surfaces"]["claude-code-desktop"]["entry_evidence"] == { + "verified": True, + "matched_files": 1, + "markers": {"claude-desktop-3p": 1}, + } + assert payload["surfaces"]["claude-code-cli"]["entry_evidence"] == { + "verified": True, + "matched_files": 1, + "markers": {"sdk-cli": 1}, + } + assert payload["native_sources"]["claude-code"]["unclassified_files"] == 1 + assert payload["surfaces"]["hermes-desktop"]["entry_evidence"]["verified"] is False + assert payload["surfaces"]["hermes-desktop"]["entry_evidence"]["matched_sessions"] == 0 + assert payload["surfaces"]["hermes-cli"]["entry_evidence"] == { + "verified": True, + "matched_sessions": 2, + "markers": {"cli": 1, "tui": 1}, + } + assert payload["surfaces"]["hermes-gateway"]["entry_evidence"]["matched_sessions"] == 1 + assert payload["surfaces"]["claude-desktop-cloud"]["automatic_capture"] is False + assert payload["surfaces"]["claude-desktop-cloud"]["capture_mode"] == "explicit-mcp-writeback" diff --git a/tests/test_memory_mcp.py b/tests/test_memory_mcp.py new file mode 100644 index 0000000..0e98d4e --- /dev/null +++ b/tests/test_memory_mcp.py @@ -0,0 +1,159 @@ +import importlib +import json +import os +from pathlib import Path +import subprocess +import sys +import types + +from scripts.config import load_config +from scripts.memory_service import MemoryService + + +class FakeFastMCP: + def __init__(self, name): + self.name = name + self.tools = {} + + def tool(self): + def register(function): + self.tools[function.__name__] = function + return function + return register + + +def service(tmp_path): + profile = tmp_path / "profile.json" + profile.write_text(json.dumps({ + "knowledge_root": str(tmp_path / "vault"), + "managed_memory_dir": "00 System/Memory", + "database_path": str(tmp_path / "memory.db"), + "skill_root": str(tmp_path / "skills"), + "skill_registry": str(tmp_path / "registry.json"), + })) + return MemoryService(load_config(profile)) + + +def load_memory_mcp(monkeypatch): + fastmcp = types.ModuleType("mcp.server.fastmcp") + fastmcp.FastMCP = FakeFastMCP + server = types.ModuleType("mcp.server") + server.fastmcp = fastmcp + mcp = types.ModuleType("mcp") + mcp.server = server + monkeypatch.setitem(sys.modules, "mcp", mcp) + monkeypatch.setitem(sys.modules, "mcp.server", server) + monkeypatch.setitem(sys.modules, "mcp.server.fastmcp", fastmcp) + sys.modules.pop("scripts.memory_mcp", None) + return importlib.import_module("scripts.memory_mcp") + + +def test_mcp_registers_seven_thin_memory_tools(tmp_path, monkeypatch): + module = load_memory_mcp(monkeypatch) + svc = service(tmp_path) + mcp = module.build_server(svc) + + assert set(mcp.tools) == { + "memory_recall", "memory_get", "memory_remember", "memory_feedback", + "memory_close_session", "memory_status", "memory_reindex", + } + + remembered = mcp.tools["memory_remember"]( + "preference", "MCP unique preference", source_hash="mcp-source", + source_ref="codex://session/turn-1", client="codex", + project_path="/tmp/project", session_id="session-1", + ) + assert remembered["ok"] is True + recalled = mcp.tools["memory_recall"]( + "MCP unique", client="claude", session_id="claude-session" + ) + assert recalled["memories"][0]["memory_id"] == remembered["memory_id"] + event = svc.store.connection.execute( + "SELECT client, session_id FROM memory_recall_events ORDER BY id DESC LIMIT 1" + ).fetchone() + assert tuple(event) == ("claude", "claude-session") + fetched = mcp.tools["memory_get"](remembered["memory_id"]) + assert fetched["memory"]["summary"] == "MCP unique preference" + assert fetched["memory"]["source_refs"] == ["codex://session/turn-1"] + assert fetched["memory"]["clients"] == ["codex"] + assert fetched["memory"]["project_path"] == "/tmp/project" + assert fetched["memory"]["session_id"] == "session-1" + + +def test_feedback_and_close_session_store_bounded_metadata(tmp_path, monkeypatch): + module = load_memory_mcp(monkeypatch) + svc = service(tmp_path) + mcp = module.build_server(svc) + remembered = svc.remember({"type": "fact", "summary": "feedback target"}) + + feedback = mcp.tools["memory_feedback"]( + remembered["memory_id"], "useful", note="helped answer" + ) + closed = mcp.tools["memory_close_session"]( + client="claude", session_id="session-1", project_path="/tmp/project", + source_ref="/tmp/session.jsonl", + ) + + assert feedback == {"ok": True, "recorded": True} + assert closed["ok"] is True + row = svc.store.connection.execute( + "SELECT client, session_id, project_path, source_ref, excerpt " + "FROM candidate_events WHERE event_id=?", (closed["event_id"],) + ).fetchone() + assert tuple(row) == ( + "claude", "session-1", "/tmp/project", "/tmp/session.jsonl", None + ) + + +def test_tool_errors_have_stable_structure(tmp_path, monkeypatch): + module = load_memory_mcp(monkeypatch) + mcp = module.build_server(service(tmp_path)) + + result = mcp.tools["memory_remember"]("invalid", "not accepted") + + assert result["ok"] is False + assert result["error"]["code"] == "validation_error" + assert isinstance(result["error"]["message"], str) + + +def test_memory_remember_exposes_supersedes_without_adding_a_tool(tmp_path, monkeypatch): + module = load_memory_mcp(monkeypatch) + svc = service(tmp_path) + mcp = module.build_server(svc) + old = svc.remember({"type": "project_state", "summary": "old MCP state"}) + + remembered = mcp.tools["memory_remember"]( + "project_state", "new MCP state", supersedes=[old["memory_id"]] + ) + invalid = mcp.tools["memory_remember"]( + "project_state", "invalid MCP state", supersedes=old["memory_id"] + ) + + assert len(mcp.tools) == 7 + assert remembered["ok"] is True + assert remembered["superseded"] == [old["memory_id"]] + assert svc.get(old["memory_id"])["status"] == "superseded" + assert invalid["ok"] is False + assert invalid["error"]["code"] == "validation_error" + + +def test_mcp_script_starts_outside_repository_cwd(tmp_path): + fake_package = tmp_path / "fake/mcp/server" + fake_package.mkdir(parents=True) + (fake_package.parent / "__init__.py").write_text("") + (fake_package / "__init__.py").write_text("") + (fake_package / "fastmcp.py").write_text( + "class FastMCP:\n" + " def __init__(self, name): pass\n" + " def tool(self): return lambda fn: fn\n" + " def run(self, transport=None): pass\n" + ) + script = Path(__file__).parents[1] / "scripts/memory_mcp.py" + env = dict(os.environ, PYTHONPATH=str(tmp_path / "fake")) + + result = subprocess.run( + [sys.executable, str(script)], cwd=str(tmp_path), env=env, + text=True, capture_output=True, + ) + + assert result.returncode == 0, result.stderr diff --git a/tests/test_memory_operations.py b/tests/test_memory_operations.py new file mode 100644 index 0000000..794788c --- /dev/null +++ b/tests/test_memory_operations.py @@ -0,0 +1,163 @@ +import json +from pathlib import Path + +from scripts.candidate_events import CandidateEventQueue +from scripts.config import load_config +from scripts.memory_operations import MemoryOperations + + +def _profile(tmp_path): + profile = tmp_path / "profile.json" + profile.write_text(json.dumps({ + "knowledge_root": str(tmp_path / "vault"), + "managed_memory_dir": "00 System/Memory", + "database_path": str(tmp_path / "memory.db"), + "skill_root": str(tmp_path / "skills"), + "skill_registry": str(tmp_path / "skills.json"), + "backup_dir": str(tmp_path / "backups"), + "backup_keep": 2, + })) + return load_config(profile) + + +def test_only_reviewed_safe_proposals_crystallize(tmp_path): + config = _profile(tmp_path) + queue = CandidateEventQueue(config.database_path) + queue.enqueue({ + "event_id": "proposal-1", "client": "codex", + "event_type": "memory_proposal", "memory_type": "decision", + "excerpt": "统一记忆只保存经过验证的结论", + "source_ref": "codex://session/turn", "confidence": "high", + "importance": 0.9, + }) + queue.enqueue({ + "event_id": "pointer-1", "client": "codex", + "event_type": "task_complete", "source_ref": "/tmp/session.jsonl#byte=1", + }) + + result = MemoryOperations(config).daily(now="2026-07-11 12:00:00") + + assert result["crystallized"] == 1 + assert result["rejected"] == 0 + rows = queue.connection.execute( + "SELECT event_id, status, memory_id FROM candidate_events ORDER BY event_id" + ).fetchall() + assert rows[0][1] == "pending" + assert rows[1][1] == "crystallized" + assert rows[1][2] + pages = list((config.knowledge_root / config.managed_memory_dir).glob("*.md")) + assert len(pages) == 1 + assert "统一记忆只保存经过验证的结论" in pages[0].read_text() + + +def test_invalid_proposal_is_rejected_without_markdown(tmp_path): + config = _profile(tmp_path) + queue = CandidateEventQueue(config.database_path) + queue.enqueue({ + "event_id": "bad", "client": "claude", + "event_type": "memory_proposal", "memory_type": "invalid", + "excerpt": "不能成为长期记忆", + }) + + result = MemoryOperations(config).daily(now="2026-07-11 12:00:00") + + assert result["rejected"] == 1 + row = queue.connection.execute( + "SELECT status, error FROM candidate_events WHERE event_id='bad'" + ).fetchone() + assert row[0] == "rejected" + assert "type" in row[1] + assert not (config.knowledge_root / config.managed_memory_dir).exists() + + +def test_daily_rejects_invalid_then_crystallizes_valid_without_locking(tmp_path): + config = _profile(tmp_path) + queue = CandidateEventQueue(config.database_path) + queue.enqueue({ + "event_id": "bad-first", "client": "claude", + "event_type": "memory_proposal", "memory_type": "invalid", + "excerpt": "不能成为长期记忆", + }) + queue.enqueue({ + "event_id": "good-second", "client": "codex", + "event_type": "memory_proposal", "memory_type": "workflow", + "excerpt": "候选拒绝后仍应继续结晶同批次的有效记忆", + "source_ref": "codex://session/verified", "confidence": "high", + "importance": 0.8, + }) + + result = MemoryOperations(config).daily(now="2026-07-12 12:00:00") + + assert result["rejected"] == 1 + assert result["crystallized"] == 1 + rows = dict(queue.connection.execute( + "SELECT event_id, status FROM candidate_events ORDER BY event_id" + ).fetchall()) + assert rows == {"bad-first": "rejected", "good-second": "crystallized"} + pages = list((config.knowledge_root / config.managed_memory_dir).glob("*.md")) + assert len(pages) == 1 + assert "同批次的有效记忆" in pages[0].read_text() + + +def test_retention_expires_pointers_and_deletes_terminal_events(tmp_path): + config = _profile(tmp_path) + queue = CandidateEventQueue(config.database_path) + queue.enqueue({"event_id": "old-pointer", "client": "codex", "event_type": "Stop"}) + queue.enqueue({"event_id": "old-done", "client": "claude", "event_type": "Stop"}) + queue.connection.execute( + "UPDATE candidate_events SET created_at='2026-05-01 00:00:00' WHERE event_id='old-pointer'" + ) + queue.connection.execute( + "UPDATE candidate_events SET status='crystallized', processed_at='2026-05-01 00:00:00' " + "WHERE event_id='old-done'" + ) + queue.connection.commit() + + result = MemoryOperations(config).daily(now="2026-07-11 12:00:00") + + assert result["expired"] == 1 + assert result["deleted"] == 1 + assert queue.connection.execute( + "SELECT status FROM candidate_events WHERE event_id='old-pointer'" + ).fetchone()[0] == "expired" + assert queue.connection.execute( + "SELECT 1 FROM candidate_events WHERE event_id='old-done'" + ).fetchone() is None + + +def test_daily_compacts_codex_turn_events_when_session_pointer_exists(tmp_path): + config = _profile(tmp_path) + queue = CandidateEventQueue(config.database_path) + queue.enqueue({ + "event_id": "session", "client": "codex", "event_type": "session_meta", + "session_id": "s1", "source_ref": "/tmp/rollout.jsonl#byte=0", + }) + queue.enqueue({ + "event_id": "turn", "client": "codex", "event_type": "turn_context", + "session_id": "s1", "source_ref": "/tmp/rollout.jsonl#byte=10", + }) + queue.enqueue({ + "event_id": "orphan", "client": "codex", "event_type": "task_complete", + "session_id": "missing", "source_ref": "/tmp/orphan.jsonl#byte=10", + }) + + result = MemoryOperations(config).daily(now="2026-07-12 12:00:00") + + assert result["compacted"] == 1 + rows = dict(queue.connection.execute( + "SELECT event_id, status FROM candidate_events" + ).fetchall()) + assert rows == {"session": "pending", "turn": "superseded", "orphan": "pending"} + + +def test_weekly_backup_rotation_and_monthly_integrity(tmp_path): + config = _profile(tmp_path) + operations = MemoryOperations(config) + for stamp in ("20260101T000000", "20260201T000000", "20260301T000000"): + result = operations.weekly(now="2026-07-11 12:00:00", backup_stamp=stamp) + assert Path(result["backup_path"]).is_file() + assert len(list(config.backup_dir.glob("memory-*.db"))) == 2 + + monthly = operations.monthly(now="2026-07-11 12:00:00", backup_stamp="20260401T000000") + assert monthly["integrity"] == "ok" + assert monthly["reindex"]["scanned"] == 0 diff --git a/tests/test_memory_service.py b/tests/test_memory_service.py new file mode 100644 index 0000000..ce07e1e --- /dev/null +++ b/tests/test_memory_service.py @@ -0,0 +1,265 @@ +import json + +import pytest + +from scripts.config import load_config +from scripts.memory_service import MemoryService, MemoryValidationError + + +def service(tmp_path): + profile = tmp_path / "profile.json" + profile.write_text(json.dumps({ + "knowledge_root": str(tmp_path / "vault"), + "managed_memory_dir": "00 System/Memory", + "database_path": str(tmp_path / "memory.db"), + "skill_root": str(tmp_path / "skills"), + "skill_registry": str(tmp_path / "registry.json"), + })) + return MemoryService(load_config(profile)) + + +def test_remember_writes_truth_and_is_immediately_recallable(tmp_path): + svc = service(tmp_path) + result = svc.remember({ + "type": "preference", "summary": "偏好唯一标识 XQ91", + "source_hash": "source-1", "confidence": "high", "importance": 0.9, + "clients": ["claude"], + }) + recalled = svc.recall("XQ91") + assert result["created"] is True + assert recalled[0]["memory_id"] == result["memory_id"] + assert list((tmp_path / "vault/00 System/Memory").glob("*.md")) + + +def test_source_hash_is_idempotent(tmp_path): + svc = service(tmp_path) + proposal = {"type": "fact", "summary": "same", "source_hash": "stable"} + first = svc.remember(proposal) + second = svc.remember(proposal) + assert second == { + "created": False, "memory_id": first["memory_id"], "superseded": [], + } + + +def test_sensitive_proposal_is_rejected(tmp_path): + svc = service(tmp_path) + with pytest.raises(MemoryValidationError): + svc.remember({"type": "fact", "summary": "password=secret123"}) + + +def test_multi_term_recall_ranks_hits_and_records_telemetry(tmp_path): + svc = service(tmp_path) + both = svc.remember({ + "type": "fact", "summary": "Claude Hermes Desktop adapter", + "importance": 0.5, + }) + svc.remember({ + "type": "fact", "summary": "Claude Desktop memory migration", + "importance": 0.9, + }) + svc.remember({ + "type": "fact", "summary": "Hermes memory adapter", + "importance": 0.8, + }) + + rows = svc.recall( + "Claude, Hermes Desktop", client="codex", session_id="session-1" + ) + + assert rows[0]["memory_id"] == both["memory_id"] + assert {row["summary"] for row in rows} == { + "Claude Hermes Desktop adapter", + "Claude Desktop memory migration", + "Hermes memory adapter", + } + event = svc.store.connection.execute( + "SELECT client, session_id, result_count, length(query_hash) " + "FROM memory_recall_events" + ).fetchone() + assert tuple(event) == ("codex", "session-1", 3, 64) + counts = svc.store.connection.execute( + "SELECT recall_count, last_recalled_at FROM memory_index" + ).fetchall() + assert all(row[0] == 1 and row[1] for row in counts) + + +def test_recall_uses_feedback_then_confidence_as_tiebreakers(tmp_path): + svc = service(tmp_path) + useful = svc.remember({ + "type": "fact", "summary": "shared ranking useful", "confidence": "low", + "importance": 0.1, + }) + high = svc.remember({ + "type": "fact", "summary": "shared ranking high", "confidence": "high", + "importance": 0.9, + }) + svc.remember({ + "type": "fact", "summary": "shared ranking medium", "confidence": "medium" + }) + svc.feedback(useful["memory_id"], "useful") + + rows = svc.recall("shared ranking") + + assert [row["summary"] for row in rows] == [ + "shared ranking useful", "shared ranking high", "shared ranking medium" + ] + + +def test_recall_rejects_empty_or_punctuation_only_query(tmp_path): + svc = service(tmp_path) + with pytest.raises(MemoryValidationError, match="query is required"): + svc.recall(" , 。 ") + + +def test_remember_supersedes_multiple_active_memories_and_hides_them_from_recall(tmp_path): + svc = service(tmp_path) + old_a = svc.remember({"type": "project_state", "summary": "obsolete alpha unique"}) + old_b = svc.remember({"type": "project_state", "summary": "obsolete beta unique"}) + + result = svc.remember({ + "type": "project_state", + "summary": "current verified state", + "supersedes": [ + old_a["memory_id"], old_b["memory_id"], old_a["memory_id"], + ], + }) + + assert result["superseded"] == [old_a["memory_id"], old_b["memory_id"]] + current = svc.get(result["memory_id"]) + assert current["supersedes"] == [old_a["memory_id"], old_b["memory_id"]] + assert current["superseded_by"] is None + for old in (old_a, old_b): + record = svc.get(old["memory_id"]) + assert record["status"] == "superseded" + assert record["superseded_by"] == result["memory_id"] + assert svc.recall("obsolete unique") == [] + + +@pytest.mark.parametrize("supersedes", ["mem_old", [""], ["missing-memory"]]) +def test_invalid_supersedes_leave_markdown_unchanged(tmp_path, supersedes): + svc = service(tmp_path) + old = svc.remember({"type": "fact", "summary": "stable old fact"}) + if supersedes == "mem_old": + supersedes = old["memory_id"] + root = tmp_path / "vault/00 System/Memory" + before = {path.name: path.read_text() for path in root.glob("*.md")} + + with pytest.raises(MemoryValidationError): + svc.remember({ + "type": "fact", "summary": "must not be written", + "supersedes": supersedes, + }) + + after = {path.name: path.read_text() for path in root.glob("*.md")} + assert after == before + + +def test_source_hash_idempotency_does_not_repeat_supersede(tmp_path): + svc = service(tmp_path) + old = svc.remember({"type": "fact", "summary": "old idempotent state"}) + proposal = { + "type": "fact", "summary": "new idempotent state", + "source_hash": "supersede-idempotency", + "supersedes": [old["memory_id"]], + } + + first = svc.remember(proposal) + old_path = tmp_path / "vault/00 System/Memory" / (old["memory_id"] + ".md") + after_first = old_path.read_text() + second = svc.remember(proposal) + + assert first["superseded"] == [old["memory_id"]] + assert second == { + "created": False, "memory_id": first["memory_id"], "superseded": [], + } + assert old_path.read_text() == after_first + + +def test_supersede_rejects_non_active_or_non_markdown_targets(tmp_path): + svc = service(tmp_path) + old = svc.remember({"type": "fact", "summary": "target lifecycle state"}) + svc.store.connection.execute( + "UPDATE memory_index SET status='archived' WHERE memory_id=?", + (old["memory_id"],), + ) + svc.store.connection.commit() + + with pytest.raises(MemoryValidationError, match="not active"): + svc.remember({ + "type": "fact", "summary": "invalid archived replacement", + "supersedes": [old["memory_id"]], + }) + + svc.store.connection.execute( + "UPDATE memory_index SET status='active', source_kind='legacy_cache' " + "WHERE memory_id=?", (old["memory_id"],), + ) + svc.store.connection.commit() + with pytest.raises(MemoryValidationError, match="not Markdown-managed"): + svc.remember({ + "type": "fact", "summary": "invalid legacy replacement", + "supersedes": [old["memory_id"]], + }) + + +def test_file_install_failure_rolls_back_all_markdown_and_index(tmp_path, monkeypatch): + svc = service(tmp_path) + old_a = svc.remember({"type": "project_state", "summary": "rollback alpha"}) + old_b = svc.remember({"type": "project_state", "summary": "rollback beta"}) + root = tmp_path / "vault/00 System/Memory" + before = {path.name: path.read_text() for path in root.glob("*.md")} + real_replace = __import__("scripts.memory_service", fromlist=["os"]).os.replace + install_count = 0 + + def fail_second_install(source, destination): + nonlocal install_count + if str(destination).endswith(".md") and str(source).endswith(".tmp"): + install_count += 1 + if install_count == 2: + raise OSError("injected install failure") + return real_replace(source, destination) + + monkeypatch.setattr("scripts.memory_service.os.replace", fail_second_install) + + with pytest.raises(OSError, match="injected install failure"): + svc.remember({ + "type": "project_state", "summary": "must roll back", + "supersedes": [old_a["memory_id"], old_b["memory_id"]], + }) + + after = {path.name: path.read_text() for path in root.glob("*.md")} + assert after == before + assert not list(root.glob(".*.tmp")) + assert not list(root.glob(".*.bak")) + assert svc.get(old_a["memory_id"])["status"] == "active" + assert svc.get(old_b["memory_id"])["status"] == "active" + + +def test_reindex_failure_restores_markdown_and_rebuilds_previous_index(tmp_path, monkeypatch): + svc = service(tmp_path) + old = svc.remember({"type": "project_state", "summary": "reindex rollback"}) + root = tmp_path / "vault/00 System/Memory" + before = {path.name: path.read_text() for path in root.glob("*.md")} + module = __import__("scripts.memory_service", fromlist=["reindex_markdown"]) + real_reindex = module.reindex_markdown + calls = 0 + + def fail_once(*args, **kwargs): + nonlocal calls + calls += 1 + if calls == 1: + raise RuntimeError("injected reindex failure") + return real_reindex(*args, **kwargs) + + monkeypatch.setattr("scripts.memory_service.reindex_markdown", fail_once) + + with pytest.raises(RuntimeError, match="injected reindex failure"): + svc.remember({ + "type": "project_state", "summary": "must not survive reindex failure", + "supersedes": [old["memory_id"]], + }) + + after = {path.name: path.read_text() for path in root.glob("*.md")} + assert calls == 2 + assert after == before + assert svc.get(old["memory_id"])["status"] == "active" diff --git a/tests/test_memory_value.py b/tests/test_memory_value.py new file mode 100644 index 0000000..c275940 --- /dev/null +++ b/tests/test_memory_value.py @@ -0,0 +1,114 @@ +import json + +from scripts.candidate_events import CandidateEventQueue +from scripts.config import load_config +from scripts.memory_value import memory_value +from scripts.unified_memory import UnifiedMemoryStore + + +def config(tmp_path): + profile = tmp_path / "profile.json" + profile.write_text(json.dumps({ + "knowledge_root": str(tmp_path / "vault"), + "database_path": str(tmp_path / "memory.db"), + "backup_dir": str(tmp_path / "backups"), + "skill_root": str(tmp_path / "skills"), + "skill_registry": str(tmp_path / "skills.json"), + })) + return load_config(profile) + + +def seed_value_data(config): + store = UnifiedMemoryStore(config.database_path) + store.connection.executemany( + "INSERT INTO memory_recall_events " + "(query_hash, client, session_id, result_count, created_at) " + "VALUES (?, ?, ?, ?, ?)", + [ + ("a" * 64, "codex", "c1", 2, "2026-07-10 12:00:00"), + ("b" * 64, "claude-desktop", "c2", 0, "2026-06-20 12:00:00"), + ("c" * 64, "hermes", "h1", 1, "2026-05-01 12:00:00"), + ], + ) + store.connection.executemany( + "INSERT INTO memory_index " + "(memory_id,type,status,summary,content_hash,clients,created_at,updated_at,source_kind) " + "VALUES (?, 'fact', 'active', ?, ?, ?, ?, ?, 'markdown')", + [ + ( + "m1", "private durable summary", "h1", '["codex"]', + "2026-07-09 12:00:00", "2026-07-09 12:00:00", + ), + ( + "m2", "older durable summary", "h2", '["hermes"]', + "2026-05-01 12:00:00", "2026-05-01 12:00:00", + ), + ], + ) + store.connection.executemany( + "INSERT INTO memory_feedback " + "(memory_id,outcome,client,created_at) VALUES (?, ?, ?, ?)", + [ + ("m1", "useful", "claude", "2026-07-10 12:00:00"), + ("m2", "misleading", "hermes-cli", "2026-06-20 12:00:00"), + ], + ) + store.connection.commit() + store.close() + + queue = CandidateEventQueue(config.database_path) + for event_id in ("h1", "h2"): + queue.enqueue({ + "event_id": event_id, + "client": "hermes-desktop", + "event_type": "session_updated", + }) + queue.connection.execute( + "UPDATE candidate_events SET created_at='2026-07-08 12:00:00'" + ) + queue.connection.commit() + queue.connection.close() + + config.backup_dir.mkdir(parents=True) + (config.backup_dir / "memory-test.db").write_bytes(b"backup") + + +def test_memory_value_aggregates_windows_clients_backlog_and_storage(tmp_path): + cfg = config(tmp_path) + seed_value_data(cfg) + + payload = memory_value(cfg, now="2026-07-11 12:00:00") + + assert payload["all_time"] == { + "recall_attempts": 3, + "recall_hits": 2, + "hit_rate": 2 / 3, + "returned_memories": 3, + "useful": 1, + "misleading": 1, + "unfeedback": 1, + "new_memories": 2, + } + assert payload["windows"]["7d"]["new_memories"] == 1 + assert payload["windows"]["7d"]["recall_attempts"] == 1 + assert payload["windows"]["30d"]["recall_attempts"] == 2 + assert payload["clients"]["hermes"]["candidate_events"] == 2 + assert payload["clients"]["hermes"]["misleading"] == 1 + assert payload["clients"]["claude"]["recall_attempts"] == 1 + assert payload["clients"]["codex"]["new_memories"] == 1 + assert payload["backlog"] == { + "pending": 2, "oldest_pending_at": "2026-07-08 12:00:00" + } + assert payload["storage"]["database_bytes"] > 0 + assert payload["storage"]["backup_bytes"] == 6 + serialized = json.dumps(payload, ensure_ascii=False) + assert "private durable summary" not in serialized + assert str(cfg.database_path) not in serialized + + +def test_memory_value_returns_zeros_without_a_database(tmp_path): + payload = memory_value(config(tmp_path), now="2026-07-11 12:00:00") + + assert payload["all_time"]["recall_attempts"] == 0 + assert payload["clients"]["codex"]["candidate_events"] == 0 + assert payload["backlog"]["oldest_pending_at"] is None diff --git a/tests/test_native_auto_extract.py b/tests/test_native_auto_extract.py new file mode 100644 index 0000000..2e11ee3 --- /dev/null +++ b/tests/test_native_auto_extract.py @@ -0,0 +1,146 @@ +import json +from pathlib import Path + +from scripts.candidate_events import CandidateEventQueue +from scripts.native_auto_extract import ( + extract_automatic_proposals, + import_native_file, + read_bounded_messages, +) + + +def test_extracts_bounded_preference_and_verified_conclusion_without_transcript(): + messages = [ + {"role": "user", "content": "记住，以后先验证再写入长期记忆，不要保存完整对话。"}, + {"role": "assistant", "content": "已验证：候选必须经过 daily 结晶，普通来源只保存指针。"}, + ] + + proposals = extract_automatic_proposals(messages) + + assert len(proposals) == 2 + assert proposals[0]["type"] == "preference" + assert "完整对话" in proposals[0]["summary"] + assert proposals[1]["type"] in {"fact", "workflow"} + assert all(len(item["summary"]) <= 800 for item in proposals) + + +def test_native_jsonl_import_reads_only_tail_and_emits_pointer_or_proposals(tmp_path, monkeypatch): + source = tmp_path / "session.jsonl" + records = [ + {"type": "user", "message": {"role": "user", "content": "private old prompt"}}, + {"type": "assistant", "message": {"role": "assistant", "content": "old tool output"}}, + {"type": "user", "message": {"role": "user", "content": "决定采用候选队列，不保存完整 transcript。"}}, + {"type": "assistant", "cwd": "/tmp/project", "message": {"role": "assistant", "content": "已验证候选队列可通过 daily 结晶。"}}, + ] + source.write_text("".join(json.dumps(item, ensure_ascii=False) + "\n" for item in records)) + queue = CandidateEventQueue(tmp_path / "memory.db") + monkeypatch.setattr(Path, "read_bytes", lambda self: (_ for _ in ()).throw( + AssertionError("native import must not read the whole file") + )) + + result = import_native_file(source, queue, client="claude-desktop", max_bytes=512) + + assert result["processed"] is True + rows = queue.connection.execute( + "SELECT event_type, excerpt FROM candidate_events ORDER BY event_id" + ).fetchall() + assert rows + assert any(row[0] == "memory_proposal" for row in rows) + assert all(row[1] is None or len(row[1]) <= 800 for row in rows) + assert "private old prompt" not in json.dumps([tuple(row) for row in rows]) + assert queue.connection.execute( + "SELECT project_path FROM candidate_events WHERE event_type='memory_proposal' LIMIT 1" + ).fetchone()[0] == "/tmp/project" + + +def test_native_import_is_idempotent_and_caps_three_proposals(tmp_path): + source = tmp_path / "session.jsonl" + source.write_text("\n".join( + json.dumps({"role": "user", "content": "以后必须记住决定 %s" % index}, ensure_ascii=False) + for index in range(5) + ) + "\n") + queue = CandidateEventQueue(tmp_path / "memory.db") + + first = import_native_file(source, queue, client="hermes") + second = import_native_file(source, queue, client="hermes") + + assert first["enqueued"] == 3 + assert second == {"processed": False, "enqueued": 0, "proposals": 0} + assert queue.connection.execute( + "SELECT COUNT(*) FROM candidate_events WHERE event_type='memory_proposal'" + ).fetchone()[0] == 3 + assert queue.get_checkpoint("native-auto:hermes:" + str(source.resolve())) is not None + + +def test_rejects_transient_decisions_memory_echoes_and_vague_status(): + messages = [ + {"role": "user", "content": "现在下午3点了，我决定待会如果不困就出去。"}, + {"role": "user", "content": "共找到 11 条记忆：ID: 9 内容: 永远优先使用某工具。"}, + {"role": "assistant", "content": "已修复并重新打开这条对话串。"}, + {"role": "assistant", "content": "人工审查发现候选里有过于模糊的“已修复”，需要加入自动提取过滤。"}, + {"role": "assistant", "content": "I've published and verified the live post."}, + {"role": "user", "content": "Web search results for query: \"FreeFound never miss automated updates\""}, + ] + + assert extract_automatic_proposals(messages) == [] + + +def test_rejects_multiline_logs_release_status_and_old_memory_self_report(): + messages = [ + {"role": "assistant", "content": "让我统计一下今天的实际工作时间：\n\n客观结论：今天工作时间确实很短。"}, + {"role": "assistant", "content": "Everything needed for the publication report is verified: same-day content was already live."}, + {"role": "assistant", "content": "全量测试通过，README 现在是双语结构。"}, + {"role": "assistant", "content": "结论：它现在能证明方向可行，但不能证明客户会买单。"}, + {"role": "assistant", "content": "DNA Memory 本机完全可用验证完成，记忆总数 72,113 条。"}, + {"role": "assistant", "content": "两个核心迁移任务已完成：\n✅ 飞书知识库已迁移\n✅ 增量更新已验证正常"}, + {"role": "assistant", "content": "你说得对，我应该先检查更多记忆文件，而不是反复问你。"}, + {"role": "assistant", "content": "已创建文件 [修图任务思考.md](修图任务思考.md)，记录了核心结论："}, + ] + + assert extract_automatic_proposals(messages) == [] + + +def test_sentence_does_not_split_inside_dot_path(): + proposals = extract_automatic_proposals([{ + "role": "assistant", + "content": "结论：根目录在 `~/.codex/sessions`，并且来源文件可读取。 后续句子不应进入摘要。", + }]) + + assert proposals[0]["summary"] == "结论：根目录在 `~/.codex/sessions`，并且来源文件可读取。" + + +def test_same_session_summary_deduplicates_across_mirrored_files(tmp_path): + queue = CandidateEventQueue(tmp_path / "memory.db") + content = json.dumps({ + "session_id": "same-session", + "messages": [{"role": "user", "content": "记住，以后不要保存完整对话。"}], + }, ensure_ascii=False) + first = tmp_path / "one.json" + second = tmp_path / "two.json" + first.write_text(content) + second.write_text(content) + + assert import_native_file(first, queue, client="claude-desktop")["enqueued"] == 1 + assert import_native_file(second, queue, client="claude-desktop")["enqueued"] == 0 + + +def test_large_hermes_json_reads_complete_messages_from_bounded_tail(tmp_path, monkeypatch): + source = tmp_path / "session_large.json" + source.write_text(json.dumps({ + "session_id": "large-session", + "system_prompt": "x" * 200000, + "messages": [ + {"role": "user", "content": "old private message"}, + {"role": "assistant", "content": "已验证：Hermes 大型 JSON 只解析有界尾部。"}, + ], + "message_count": 2, + }, ensure_ascii=False)) + monkeypatch.setattr(Path, "read_text", lambda self, *args, **kwargs: (_ for _ in ()).throw( + AssertionError("bounded reader must not read the whole JSON file") + )) + + messages = read_bounded_messages(source, max_bytes=4096) + + assert messages[-1]["role"] == "assistant" + assert "有界尾部" in messages[-1]["content"] + assert sum(len(message["content"]) for message in messages) < 4096 diff --git a/tests/test_native_history.py b/tests/test_native_history.py new file mode 100644 index 0000000..557d768 --- /dev/null +++ b/tests/test_native_history.py @@ -0,0 +1,113 @@ +import json + +from scripts.candidate_events import CandidateEventQueue +from scripts.import_native_history import SourceSpec, import_paths, prune_obsolete_sources + + +def test_history_import_scans_supported_sources_with_file_budget(tmp_path): + codex = tmp_path / "codex" + hermes = tmp_path / "hermes" + codex.mkdir() + hermes.mkdir() + (codex / "a.jsonl").write_text(json.dumps({ + "type": "response_item", "payload": { + "type": "message", "role": "assistant", + "content": [{"type": "output_text", "text": "已验证：Codex 规则生效。"}], + }, + }) + "\n") + (hermes / "b.jsonl").write_text(json.dumps({ + "role": "user", "content": "以后必须先检查再发布。", + }) + "\n") + queue = CandidateEventQueue(tmp_path / "memory.db") + + result = import_paths( + {"codex": [codex], "hermes": [hermes]}, queue, + max_files=1, min_age_seconds=0, + ) + + assert result["files"] == 2 + assert result["processed"] == 2 + assert result["proposals"] == 2 + + second = import_paths( + {"codex": [codex], "hermes": [hermes]}, queue, + max_files=1, min_age_seconds=0, + ) + assert second == {"files": 0, "processed": 0, "enqueued": 0, "proposals": 0} + + +def test_history_import_uses_explicit_client_source_specs(tmp_path): + claude = tmp_path / "claude" + desktop = tmp_path / "desktop" + hermes = tmp_path / "hermes" + for path in (claude, desktop, hermes): + path.mkdir() + message = json.dumps({"role": "user", "content": "以后必须先验证再发布。"}) + "\n" + (claude / "real.jsonl").write_text(message) + (claude / "tool-result.json").write_text(message) + (desktop / "audit.jsonl").write_text(message) + (desktop / "other.jsonl").write_text(message) + (hermes / "session_live.json").write_text(json.dumps({ + "session_id": "live", "messages": [ + {"role": "user", "content": "以后必须先验证再发布。"}, + ], + })) + (hermes / "request_dump_private.json").write_text(message) + queue = CandidateEventQueue(tmp_path / "memory.db") + + result = import_paths({ + "claude-code": SourceSpec((claude,), ("*.jsonl",)), + "claude-desktop": SourceSpec((desktop,), ("audit.jsonl",)), + "hermes": SourceSpec((hermes,), ("*.jsonl", "session_*.json")), + }, queue, max_files=10, min_age_seconds=0) + + assert result["files"] == 3 + sources = { + row[0] for row in queue.connection.execute( + "SELECT DISTINCT source_ref FROM candidate_events" + ).fetchall() + } + assert not any("tool-result.json" in source for source in sources) + assert not any("other.jsonl" in source for source in sources) + assert not any("request_dump" in source for source in sources) + + +def test_prune_obsolete_removes_only_auto_events_for_invalid_sources(tmp_path): + root = tmp_path / "claude" + root.mkdir() + valid = root / "valid.jsonl" + invalid = root / "tool-result.json" + valid.write_text("{}\n") + invalid.write_text("{}\n") + queue = CandidateEventQueue(tmp_path / "memory.db") + for path in (valid, invalid): + stat = path.stat() + queue.update_checkpoint( + "native-auto:claude-code:" + str(path.resolve()), + stat.st_ino, stat.st_size, "digest", + ) + queue.enqueue({ + "event_id": "auto_session_" + path.stem, + "client": "claude-code", + "event_type": "session_updated", + "source_ref": str(path.resolve()), + "source_hash": "digest", + }) + queue.enqueue({ + "event_id": "manual-pointer", + "client": "claude-code", + "event_type": "session_updated", + "source_ref": str(invalid.resolve()), + "source_hash": "manual", + }) + + result = prune_obsolete_sources({ + "claude-code": SourceSpec((root,), ("*.jsonl",)), + }, queue) + + assert result == {"checkpoints": 1, "events": 1} + assert queue.get_checkpoint("native-auto:claude-code:" + str(valid.resolve())) + assert queue.get_checkpoint("native-auto:claude-code:" + str(invalid.resolve())) is None + assert queue.connection.execute( + "SELECT COUNT(*) FROM candidate_events WHERE event_id='manual-pointer'" + ).fetchone()[0] == 1 diff --git a/tests/test_policy.py b/tests/test_policy.py new file mode 100644 index 0000000..3c78cb1 --- /dev/null +++ b/tests/test_policy.py @@ -0,0 +1,25 @@ +import pytest + +from scripts.policy import capacity_status, inspect_content + + +@pytest.mark.parametrize("content", [ + "sk-" + "abcdefghijklmnopqrstuvwxyz123456", + "-----BEGIN " + "PRIVATE KEY-----", + "password=secret123", + "A" * 6000, +]) +def test_sensitive_content_is_rejected(content): + result = inspect_content(content) + assert result.allowed is False + assert result.reason + + +def test_capacity_hard_limit_blocks_writes(tmp_path): + db = tmp_path / "memory.db" + db.write_bytes(b"12") + + status = capacity_status(db, warning_bytes=1, hard_bytes=2) + + assert status.state == "blocked" + assert status.writable is False diff --git a/tests/test_public_safety.py b/tests/test_public_safety.py new file mode 100644 index 0000000..1534854 --- /dev/null +++ b/tests/test_public_safety.py @@ -0,0 +1,25 @@ +from pathlib import Path + +from scripts.check_public_safety import inspect_tree + + +def test_public_safety_accepts_generic_examples(tmp_path): + (tmp_path / "profile.json").write_text( + '{"knowledge_root":"~/Documents/DNA-Memory-Vault"}', encoding="utf-8" + ) + + assert inspect_tree(tmp_path) == [] + + +def test_public_safety_rejects_private_paths_and_tokens(tmp_path): + private_root = "/" + "Users/private-person/Desktop/private-vault" + token = "sk-" + "abcdefghijklmnopqrstuvwxyz123456" + (tmp_path / "config.txt").write_text( + "root={}\ntoken={}".format(private_root, token), encoding="utf-8" + ) + + findings = inspect_tree(tmp_path) + + assert {finding[1] for finding in findings} == { + "private identifier", "OpenAI-style token" + } diff --git a/tests/test_skill_manager.py b/tests/test_skill_manager.py new file mode 100644 index 0000000..742d0ce --- /dev/null +++ b/tests/test_skill_manager.py @@ -0,0 +1,54 @@ +import json + +from scripts.skill_manager import build_sync_plan, inventory + + +def skill(root, name, text): + path = root / name + path.mkdir(parents=True) + (path / "SKILL.md").write_text(text) + return path + + +def layout(tmp_path): + shared = tmp_path / "shared" + codex = tmp_path / "codex" + claude = tmp_path / "claude" + shared.mkdir() + codex.mkdir() + claude.mkdir() + source = skill(shared, "shared-ok", "---\nname: shared-ok\n---\n") + (codex / "shared-ok").symlink_to(source, target_is_directory=True) + skill(claude, "shared-ok", "different") + skill(claude, "claude-only", "platform") + (codex / "broken-link").symlink_to(tmp_path / "missing", target_is_directory=True) + registry = { + "skills": { + "shared-ok": {"targets": ["codex", "claude"]}, + "missing-target": {"targets": ["codex"]}, + } + } + return shared, {"codex": codex, "claude": claude}, registry + + +def test_inventory_classifies_shared_conflict_platform_and_broken(tmp_path): + shared, roots, registry = layout(tmp_path) + + findings = inventory(shared, roots, registry) + states = {(item.name, item.platform): item.state for item in findings} + + assert states[("shared-ok", "codex")] == "shared" + assert states[("shared-ok", "claude")] == "conflict" + assert states[("claude-only", "claude")] == "platform" + assert states[("broken-link", "codex")] == "broken_link" + + +def test_sync_plan_never_overwrites_conflicts(tmp_path): + shared, roots, registry = layout(tmp_path) + + plan = build_sync_plan(shared, roots, registry) + actions = {(item.name, item.platform): item.action for item in plan} + + assert actions[("shared-ok", "codex")] == "ok" + assert actions[("shared-ok", "claude")] == "blocked_conflict" + assert actions[("missing-target", "codex")] == "missing_source" diff --git a/tests/test_skills_cli.py b/tests/test_skills_cli.py new file mode 100644 index 0000000..0e46107 --- /dev/null +++ b/tests/test_skills_cli.py @@ -0,0 +1,31 @@ +import json + +from scripts.skills_cli import main + + +def test_sync_defaults_to_dry_run_and_apply_creates_only_missing_links(tmp_path, capsys): + shared = tmp_path / "shared" + source = shared / "one" + source.mkdir(parents=True) + (source / "SKILL.md").write_text("---\nname: one\n---\n") + codex = tmp_path / "codex" + registry = tmp_path / "skills.json" + registry.write_text(json.dumps({"skills": {"one": {"targets": ["codex"]}}})) + profile = tmp_path / "profile.json" + profile.write_text(json.dumps({ + "knowledge_root": str(tmp_path / "vault"), + "database_path": str(tmp_path / "memory.db"), + "skill_root": str(shared), + "skill_registry": str(registry), + "platform_skill_roots": {"codex": str(codex)}, + })) + + assert main(["--profile", str(profile), "sync", "--json"]) == 0 + dry = json.loads(capsys.readouterr().out) + assert dry["changed"] == 0 + assert (codex / "one").exists() is False + + assert main(["--profile", str(profile), "sync", "--apply", "--json"]) == 0 + applied = json.loads(capsys.readouterr().out) + assert applied["changed"] == 1 + assert (codex / "one").resolve() == source.resolve() diff --git a/tests/test_unified_memory.py b/tests/test_unified_memory.py new file mode 100644 index 0000000..a69519d --- /dev/null +++ b/tests/test_unified_memory.py @@ -0,0 +1,75 @@ +import sqlite3 + +from scripts.unified_memory import UnifiedMemoryStore + + +def create_legacy_db(path, content="保留我"): + conn = sqlite3.connect(str(path)) + conn.execute(""" + CREATE TABLE memory ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + content TEXT NOT NULL, + type TEXT DEFAULT 'fact', + tags TEXT DEFAULT '', + weight REAL DEFAULT 0.5, + short_term INTEGER DEFAULT 1, + long_term INTEGER DEFAULT 0, + created REAL, + updated REAL, + last_accessed REAL + ) + """) + conn.execute( + "INSERT INTO memory (content, type, weight, created, updated) VALUES (?, 'fact', 0.7, 1, 2)", + (content,), + ) + conn.commit() + conn.close() + return path + + +def test_migration_preserves_legacy_rows_and_is_idempotent(tmp_path): + db = create_legacy_db(tmp_path / "memory.db") + store = UnifiedMemoryStore(db) + store.migrate() + store.migrate() + + row = store.connection.execute( + "SELECT summary, legacy_id, source_kind FROM memory_index" + ).fetchone() + count = store.connection.execute("SELECT COUNT(*) FROM memory_index").fetchone()[0] + + assert tuple(row) == ("保留我", 1, "legacy_cache") + assert count == 1 + store.close() + + +def test_schema_supports_source_refs_and_memory_relationships(tmp_path): + store = UnifiedMemoryStore(tmp_path / "memory.db") + columns = { + row[1] for row in store.connection.execute("PRAGMA table_info(memory_index)").fetchall() + } + tables = { + row[0] for row in store.connection.execute( + "SELECT name FROM sqlite_master WHERE type='table'" + ).fetchall() + } + version = store.connection.execute("SELECT MAX(version) FROM schema_version").fetchone()[0] + + assert "source_refs" in columns + assert {"supersedes", "superseded_by"}.issubset(columns) + assert "memory_links" in tables + assert "memory_feedback" in tables + recall_columns = { + row[1] for row in store.connection.execute( + "PRAGMA table_info(memory_recall_events)" + ).fetchall() + } + + assert "recall_count" in columns + assert "memory_recall_events" in tables + assert { + "query_hash", "client", "session_id", "result_count", "created_at" + }.issubset(recall_columns) + assert version == 5 + store.close()