diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 0000000..90b5d8c --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,93 @@ +{ + "name": "dspy.ts", + "metadata": { + "version": "0.1.0", + "description": "DSPy.ts Claude Code & Codex plugins — program AI systems on AgentDB. From practical (scaffold/compile/eval) to exotic (GEPA self-evolution) plus vertical appliances." + }, + "owner": { + "name": "rUv", + "url": "https://github.com/ruvnet" + }, + "plugins": [ + { + "name": "dspy-core", + "source": "./plugins/dspy-core", + "description": "Scaffold, compile (BootstrapFewShot/MIPROv2/GEPA) and evaluate DSPy.ts programs; the `dspy-architect` agent designs signatures, modules and metrics.", + "version": "0.1.0", + "category": "official", + "status": "available" + }, + { + "name": "dspy-optimize", + "source": "./plugins/dspy-optimize", + "description": "Deep optimizer workflows: MIPROv2 + experience replay, GEPA Pareto evolution, BootstrapFewShot dynamic demos; `optimizer-engineer` agent.", + "version": "0.1.0", + "category": "official", + "status": "available" + }, + { + "name": "dspy-rag", + "source": "./plugins/dspy-rag", + "description": "RAG with `RetrieveModule` over AgentDB (HNSW, RaBitQ, MMR) wired into ChainOfThought; `rag-architect` agent; corpus-indexing commands.", + "version": "0.1.0", + "category": "official", + "status": "available" + }, + { + "name": "dspy-react", + "source": "./plugins/dspy-react", + "description": "ReAct agents with tool registries + `ReActReflexion` (recall lessons, record episodes, promote skills); `react-engineer` agent.", + "version": "0.1.0", + "category": "official", + "status": "available" + }, + { + "name": "dspy-observability", + "source": "./plugins/dspy-observability", + "description": "`CompilationTracer` causal-chain traces, AgentDB persistence, optional MLflow; `CachingLM` setup; `observability-engineer` agent.", + "version": "0.1.0", + "category": "official", + "status": "available" + }, + { + "name": "dspy-evolution", + "source": "./plugins/dspy-evolution", + "description": "Exotic: GEPA-driven self-evolution loops that optimize a program's prompts (and the program itself) against a benchmark across runs; `evolution-coordinator` agent.", + "version": "0.1.0", + "category": "exotic", + "status": "available" + }, + { + "name": "dspy-appliance-support-bot", + "source": "./plugins/dspy-appliance-support-bot", + "description": "Vertical appliance: a pre-wired DSPy.ts support assistant — RAG over a knowledge base + ChainOfThought answer + a quality metric + MIPROv2 tuning.", + "version": "0.1.0", + "category": "appliance", + "status": "available" + }, + { + "name": "dspy-appliance-code-review", + "source": "./plugins/dspy-appliance-code-review", + "description": "Vertical appliance: a DSPy.ts code-review pipeline — Retrieve(repo context) → ChainOfThought(review) → metric on actionability; GEPA-tuned.", + "version": "0.1.0", + "category": "appliance", + "status": "available" + }, + { + "name": "dspy-appliance-research-assistant", + "source": "./plugins/dspy-appliance-research-assistant", + "description": "Vertical appliance: a DSPy.ts research assistant — ReAct(search tools)+reflexion → synthesize → cite; optimized for grounded answers.", + "version": "0.1.0", + "category": "appliance", + "status": "available" + }, + { + "name": "dspy-appliance-data-pipeline", + "source": "./plugins/dspy-appliance-data-pipeline", + "description": "Vertical appliance: a DSPy.ts ETL/classification pipeline — typed Predict modules in a Pipeline, BootstrapFewShot from a labeled CSV, evaluated batch.", + "version": "0.1.0", + "category": "appliance", + "status": "available" + } + ] +} diff --git a/README.md b/README.md index 0cabd6d..900e21e 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,35 @@ await opt.compile(qa, trainset); --- +## Plugins (Claude Code & Codex) + +DSPy.ts ships a **plugin marketplace** so you can program, compile, evaluate, and run DSPy.ts programs from inside Claude Code or OpenAI Codex — commands, sub-agents, design skills, and bundled MCP servers exposing the library as tools/resources. See [`docs/adr/ADR-0001-claude-code-plugin-marketplace.md`](./docs/adr/ADR-0001-claude-code-plugin-marketplace.md). + +```bash +# add the marketplace, then install a plugin +/plugin marketplace add ruvnet/dspy.ts +/plugin install dspy-core@dspy.ts +# or try one locally from a checkout +claude --plugin-dir ./plugins/dspy-core +``` + +| Plugin | What it does | +|--------|--------------| +| `dspy-core` | Scaffold / compile (BootstrapFewShot · MIPROv2 · GEPA) / evaluate programs; `dspy-architect` agent; signature- & metric-design skills; MCP tools for the library | +| `dspy-optimize` | Deep optimizer workflows — MIPROv2 + experience replay, GEPA Pareto evolution, BootstrapFewShot dynamic demos; `optimizer-engineer` agent | +| `dspy-rag` | `RetrieveModule` (MMR) over an AgentDB corpus → `ChainOfThought`, grounded + cited; corpus indexing; `rag-architect` agent | +| `dspy-react` | ReAct agents + tool registries + `ReActReflexion` (recall lessons, record episodes, promote skills); `react-engineer` agent | +| `dspy-observability` | `CompilationTracer` causal-chain traces (AgentDB / optional MLflow) + `CachingLM`; `observability-engineer` agent | +| `dspy-evolution` *(exotic)* | Multi-generation GEPA self-evolution against a held-out benchmark — persistent Pareto frontier, warm-start, optional structural exploration; `evolution-coordinator` agent | +| `dspy-appliance-support-bot` | Vertical appliance: a pre-wired RAG support assistant (Retrieve → CoT + citations + quality metric + MIPROv2 tuning) | +| `dspy-appliance-code-review` | Vertical appliance: a code-review pipeline (Retrieve repo context → CoT structured review + actionability metric + GEPA tuning) | +| `dspy-appliance-research-assistant` | Vertical appliance: a ReAct(search/fetch/note)+reflexion → CoT synthesizer that writes grounded, cited answers | +| `dspy-appliance-data-pipeline` | Vertical appliance: typed `PredictModule` stages in a `Pipeline` + CSV/JSONL batch I/O + BootstrapFewShot from a labeled CSV | + +Source: [`plugins/`](./plugins/) · marketplace manifest: [`.claude-plugin/marketplace.json`](./.claude-plugin/marketplace.json) + +--- + ## Documentation | Doc | Where | diff --git a/docs/adr/ADR-0001-claude-code-plugin-marketplace.md b/docs/adr/ADR-0001-claude-code-plugin-marketplace.md new file mode 100644 index 0000000..fec1084 --- /dev/null +++ b/docs/adr/ADR-0001-claude-code-plugin-marketplace.md @@ -0,0 +1,89 @@ +# ADR-0001: DSPy.ts Claude Code & Codex Plugin Marketplace + +- **Status:** Accepted +- **Date:** 2026-05-11 +- **Deciders:** rUv +- **Tags:** plugins, marketplace, mcp, claude-code, codex, dx + +## Context + +DSPy.ts ships a real programming model — typed `Signature`s, modules (`PredictModule`, +`ChainOfThought`, `ReAct`, `RetrieveModule`, `Pipeline`), optimizers +(`BootstrapFewShot`, `MIPROv2`, `GEPA`), and an AgentDB memory layer (HNSW + RaBitQ + +MMR, ReasoningBank, ReActReflexion, CompilationTracer). Today a user has to read the +docs/examples and wire all of that by hand. We want the library to be *usable from +inside an agent* — Claude Code and OpenAI Codex — so an agent can scaffold a program, +compile it with an optimizer, evaluate it, and run RAG/ReAct without the human +hand-assembling the pieces. + +Claude Code supports **plugins** (`.claude-plugin/plugin.json` + `commands/`, +`agents/`, `skills/`, and bundled `mcpServers`), distributed through a +**marketplace** (`.claude-plugin/marketplace.json`). Codex consumes the same MCP +servers and reads `AGENTS.md`. So a single repo can publish: a marketplace manifest, +several plugins, and MCP tools/resources that expose the library itself. + +## Decision + +Publish a **plugin marketplace from the `ruvnet/dspy.ts` repo**: + +1. **`.claude-plugin/marketplace.json`** at the repo root lists all plugins. +2. **`plugins//`** — one directory per plugin, each a valid Claude Code plugin + (`.claude-plugin/plugin.json`, `commands/*.md`, `agents/*.md`, + `skills//SKILL.md`), and where useful a bundled **MCP server** (`mcp/server.js`, + referenced from `plugin.json`'s `mcpServers`) exposing DSPy.ts as tools + (`dspy_scaffold`, `dspy_compile`, `dspy_eval`, `dspy_retrieve`, …) and resources + (`dspy://docs/api`, `dspy://examples`, the design-skill guides). Codex picks up the + same MCP servers. +3. **Tiers, practical → exotic, plus vertical appliances:** + - `dspy-core` — scaffold / compile / evaluate; `dspy-architect` agent; signature- & + metric-design skills; MCP tools for the library. *(shipped first)* + - `dspy-optimize` — deep optimizer workflows (MIPROv2 + experience replay, GEPA + Pareto evolution, BootstrapFewShot dynamic demos). + - `dspy-rag` — `RetrieveModule` over AgentDB (HNSW/RaBitQ/MMR) → ChainOfThought; + corpus indexing. + - `dspy-react` — ReAct + tool registries + `ReActReflexion` (recall lessons, record + episodes, promote skills). + - `dspy-observability` — `CompilationTracer` causal traces, AgentDB persistence, + optional MLflow; `CachingLM`. + - `dspy-evolution` — *(exotic)* GEPA-driven self-evolution loops that evolve a + program's prompts (and the program) against a benchmark across runs. + - **Vertical appliances** — pre-wired end-to-end programs: + `dspy-appliance-support-bot`, `dspy-appliance-code-review`, + `dspy-appliance-research-assistant`, `dspy-appliance-data-pipeline`. +4. **Build cadence:** `dspy-core` + the marketplace manifest land now; the remaining + plugins are built one per iteration via a recurring loop, each scaffolded, validated + (plugin.json parses, frontmatter present, referenced files exist) and committed, + tracked in a GitHub issue, finishing with a single PR. + +## Consequences + +**Positive** +- The library becomes operable from Claude Code / Codex with no hand-wiring; agents get + first-class commands, sub-agents, design skills, and MCP tools/resources. +- One source of truth (this repo) for the marketplace; users add it with + `/plugin marketplace add ruvnet/dspy.ts`. +- Vertical appliances give a working baseline you can `dspy-compile` further — they + double as examples. + +**Negative / risks** +- More surface to keep in sync with the `dspy.ts` API — mitigated by pinning + `minClaudeFlowVersion`-style version hints in each `plugin.json` and keeping command + bodies thin (they orchestrate, the library does the work). +- The bundled MCP servers start as scaffolds (tool schemas + handler stubs that shell + out to `npx ts-node`) — fleshing out the stdio transport + handlers is follow-up work + per plugin. + +## Alternatives considered + +- **Docs only / no plugins** — rejected: doesn't make the library agent-operable. +- **One mega-plugin** — rejected: a 12-field-signature problem; tiers + appliances are + clearer and let users install only what they need. +- **A separate `dspy-plugins` repo** — rejected: keeps the marketplace away from the + code it wraps; co-locating means a PR can change both together. + +## References + +- Tracking issue: roadmap + per-plugin checklist on `ruvnet/dspy.ts`. +- `dspy.ts@2.2.0` API: `configureLM`, `ChainOfThought`, `BootstrapFewShot`, `MIPROv2`, + `GEPA`, `RetrieveModule`, `AgentDBClient`, `ReActReflexion`, `CachingLM`, + `CompilationTracer`. diff --git a/plugins/dspy-appliance-code-review/.claude-plugin/plugin.json b/plugins/dspy-appliance-code-review/.claude-plugin/plugin.json new file mode 100644 index 0000000..f3d65d8 --- /dev/null +++ b/plugins/dspy-appliance-code-review/.claude-plugin/plugin.json @@ -0,0 +1,10 @@ +{ + "name": "dspy-appliance-code-review", + "description": "Vertical appliance: a pre-wired DSPy.ts code-review pipeline. Ships a ready-to-run program — RetrieveModule pulls relevant repo context (conventions, related code, prior reviews) from an AgentDB index → ChainOfThought produces a structured review (severity-tagged findings + suggested fixes) → an actionability metric scores it → GEPA tuning. Commands scaffold it into your repo, review a diff/file, and tune the reviewer.", + "version": "0.1.0", + "author": { "name": "rUv", "url": "https://github.com/ruvnet" }, + "homepage": "https://github.com/ruvnet/dspy.ts", + "license": "MIT", + "keywords": ["dspy", "dspy.ts", "appliance", "code-review", "rag", "gepa", "agentdb", "pull-request"], + "mcpServers": { "dspy-appliance-code-review": { "command": "node", "args": ["${CLAUDE_PLUGIN_ROOT}/mcp/server.js"] } } +} diff --git a/plugins/dspy-appliance-code-review/AGENTS.md b/plugins/dspy-appliance-code-review/AGENTS.md new file mode 100644 index 0000000..25c4e65 --- /dev/null +++ b/plugins/dspy-appliance-code-review/AGENTS.md @@ -0,0 +1,13 @@ +# dspy-appliance-code-review — for Codex / MCP clients + +A vertical appliance: a pre-wired DSPy.ts code-review pipeline. Ships a ready-to-run program (`templates/code-reviewer.ts` — `RetrieveModule` pulls repo context → `ChainOfThought` produces a structured review + an `actionabilityMetric`) plus tooling to stand it up. MCP server: `mcp/server.js`, command `node ${CLAUDE_PLUGIN_ROOT}/mcp/server.js`. + +**Tools** +- `code_review_init` — copy the appliance program into `src/dspy/code-reviewer.ts` (+ spec), build the AgentDB repo-context index (conventions, ADRs, representative modules, prior PR reviews). +- `code_review_run` — review a diff/file: retrieves relevant repo context, returns `{summary, findings:[{severity,location,issue,suggestion}], questions, passages, context}`. +- `code_review_tune` — GEPA-tune (or MIPROv2) the reviewer against a labelled set (`[{input:{diff,intent?}, output:{knownIssues:[{severity,near}], verdict}}]`) and the actionability metric (coverage − noise, specificity, calibration; false alarms on clean PRs penalised); saves `.gepa.json`. +- `code_review_status` — context-index stats + whether a tuned reviewer is loaded. + +**Resources**: `dspy://code-reviewer/template`, `dspy://review-context-indexing`, `dspy://review-actionability-metric`. + +Handlers shell out to `npx ts-node` against `src/dspy/*` (the copied appliance) and `dspy.ts`. Slash commands `/code-review-init`, `/code-review-run`, `/code-review-tune` wrap the same flows; the `code-review-builder` agent stands up and tunes a reviewer for a specific repo. Builds on `dspy-rag` (index/retrieve), `dspy-optimize` + `dspy-evolution` (GEPA/MIPROv2 tuning), `dspy-observability` (tracer, cache). diff --git a/plugins/dspy-appliance-code-review/agents/code-review-builder.md b/plugins/dspy-appliance-code-review/agents/code-review-builder.md new file mode 100644 index 0000000..563fc08 --- /dev/null +++ b/plugins/dspy-appliance-code-review/agents/code-review-builder.md @@ -0,0 +1,14 @@ +--- +name: code-review-builder +description: Stands up and tunes the DSPy.ts code-review appliance for a specific repo — scaffolds the program, builds the repo-context index (conventions, representative code, ADRs, prior reviews), assembles a labelled review set, GEPA-tunes the reviewer against the actionability metric, and validates review quality. Use to go from "review PRs against our conventions" to a tuned reviewer. +--- +You build code-review appliances on DSPy.ts. + +STEPS: +1. **Scaffold** — `/code-review-init --context --include '' --prior-reviews `. The appliance code (`RetrieveModule → ChainOfThought` + `actionabilityMetric`) is the starting point; keep the prompt's hard constraints — *judge against the repo's actual conventions (cite them), every finding is located + has a concrete fix, severity-tag honestly, say "ship it" when it's good*. +2. **Index the right context** — conventions docs, ADRs, a *representative sample* of modules (the patterns you want enforced), and especially **prior PR reviews** (they encode what this team actually flags). Metadata `{ source, kind }`. Don't bulk-index the whole repo — that dilutes retrieval; curate. (See the `review-context-indexing` skill.) +3. **Labelled review set** — from past PRs: `{ input:{diff, intent}, output:{knownIssues:[{severity, near}], verdict} }`. **Include clean PRs** (`verdict: 'ship'`, no `knownIssues`) so tuning penalises false alarms — without them the reviewer learns to always find problems. Hold out a slice. +4. **Tune** — `/code-review-tune ` with GEPA against `actionabilityMetric` (coverage of real issues − noise, specificity, severity calibration), `frontierStore` + a `CachingLM`. Read the reflections — which instruction change made it more specific / less noisy? For ongoing evolution, hand to `dspy-evolution`'s `/dspy-evolve`. +5. **Validate** — `/code-review-run` on held-out diffs: does it catch the known issues? are findings located + actionable? does it stay quiet on clean PRs? does it cite real conventions (not invented ones)? (See the `review-actionability-metric` skill.) + +DELIVER: the program file, the context index, the labelled set, the tuned reviewer (`.gepa.json`), and a quality report (mean metric, examples of good findings, false-alarm rate on clean PRs, recall on known issues). diff --git a/plugins/dspy-appliance-code-review/commands/code-review-init.md b/plugins/dspy-appliance-code-review/commands/code-review-init.md new file mode 100644 index 0000000..a063958 --- /dev/null +++ b/plugins/dspy-appliance-code-review/commands/code-review-init.md @@ -0,0 +1,11 @@ +--- +description: Scaffold the DSPy.ts code-review appliance into your repo — copy the program template to src/dspy/code-reviewer.ts (+ spec), build the AgentDB repo-context index (conventions, representative modules, ADRs, prior reviews), and print next steps. +argument-hint: "[--dest src/dspy/code-reviewer.ts] [--context .dspy/review-context] [--include 'src/**/*.ts,docs/adr/**,CONVENTIONS.md'] [--prior-reviews path/to/reviews.json]" +--- +Set up the code-review appliance. Parse `$ARGUMENTS` for `--dest` (default `src/dspy/code-reviewer.ts`), `--context` (AgentDB index path, default `.dspy/review-context`), `--include` (globs of repo material to index: conventions docs, ADRs, a sample of representative modules), `--prior-reviews` (a JSON of past PR review comments to index — optional but high value). + +1. Confirm `dspy.ts` is a dependency. +2. Copy `${CLAUDE_PLUGIN_ROOT}/templates/code-reviewer.ts` → `--dest` (+ `code-reviewer.spec.ts`); rewrite `CONTEXT_PATH` to `--context`. +3. `const ctx = new AgentDBClient({ vectorDimension: 384, storage: { path: contextPath } }); await ctx.init();` +4. Index the `--include` material: chunk each file (keep `{ source, kind: 'convention'|'adr'|'code'|'review' }` metadata), `ctx.storeText(chunk, meta, { tier: 'long' })`. Index `--prior-reviews` too (`kind: 'review'`) — past reviews teach the reviewer what this team flags. Don't index the whole repo — a representative sample of conventions + patterns + reviews is what helps; bulk source just dilutes retrieval. +5. Print next steps: `git diff` then `/code-review-run` on it; build `tune/reviews.json` (`[{ input:{diff, intent?}, output:{knownIssues:[{severity, near}], verdict} }]` from labelled past PRs); then `/code-review-tune tune/reviews.json`. diff --git a/plugins/dspy-appliance-code-review/commands/code-review-run.md b/plugins/dspy-appliance-code-review/commands/code-review-run.md new file mode 100644 index 0000000..bdbd226 --- /dev/null +++ b/plugins/dspy-appliance-code-review/commands/code-review-run.md @@ -0,0 +1,11 @@ +--- +description: Run the code-review appliance on a diff or file — retrieves relevant repo context (conventions, related code, prior reviews), produces a structured review (severity-tagged, located findings + suggested fixes), and prints it. +argument-hint: " [--intent \"PR title/description\"] [--program src/dspy/code-reviewer.ts] [--severity-min nit|minor|major|blocker] [--show-context]" +--- +Review a change. Parse `$ARGUMENTS` for the target (a path to a `.diff`/`.patch`, a source file, or `-` to read a diff from stdin / `git diff`), `--intent` (what the change is meant to do), `--program`, `--severity-min` (filter findings), `--show-context`. + +1. Obtain the diff (read the file, or `git diff` if `-`). +2. `const rev = await buildCodeReviewer();` (from the appliance module) — `RetrieveModule → ChainOfThought`. +3. `const out = await rev.run({ diff, intent });` +4. Print: the **summary** (ship / changes-needed / blocked + why); the **findings** grouped by severity — each `location · issue · suggestion` (filter by `--severity-min`); the **questions** for the author; and the **passages** that informed it (rank · score · source · kind). With `--show-context`, the assembled context. +5. If findings are vague ("consider refactoring") or it flags things that aren't actually project conventions: that's what `/code-review-tune` fixes (the metric rewards specific, located, convention-cited findings and penalises noise). If it misses obvious issues: index more representative code / prior reviews into the context. diff --git a/plugins/dspy-appliance-code-review/commands/code-review-tune.md b/plugins/dspy-appliance-code-review/commands/code-review-tune.md new file mode 100644 index 0000000..6c12ca8 --- /dev/null +++ b/plugins/dspy-appliance-code-review/commands/code-review-tune.md @@ -0,0 +1,11 @@ +--- +description: Tune the code-review appliance's reviewer with GEPA (reflective Pareto evolution) against labelled past reviews and the actionability metric — so it catches the real issues, stays specific, and doesn't false-alarm on clean PRs. +argument-hint: " [--program src/dspy/code-reviewer.ts] [--iterations N] [--frontier .dspy/review-frontier] [--cache .dspy/review-cache] [--mipro]" +--- +Optimize the `reviewer` `ChainOfThought` in `src/dspy/code-reviewer.ts`. Parse `$ARGUMENTS` for the labelled set (`[{ input:{diff, intent?}, output:{knownIssues:[{severity, near}], verdict:'ship'|'changes'|'blocked'} }]`), `--program`, `--iterations` (GEPA iterations, default 12), `--frontier` (AgentDB path → `frontierStore`, persists/continues the evolution), `--cache` (AgentDB path → `CachingLM`), `--mipro` (use MIPROv2 instead of GEPA — faster, less aggressive). + +1. Build a *training* program: `RetrieveModule(context) → reviewer`, so each example is scored end-to-end (context retrieval + review) via `actionabilityMetric`. +2. Optional: `configureLM(new CachingLM(getLM(), { store: cacheClient, similarityThreshold: 0.985, embed: 'model' }))` — the search makes many near-identical prompts. +3. `--mipro`: `new MIPROv2(actionabilityMetric, { numTrials: iterations, replayStore: store, tracer: new CompilationTracer({ store }) })`. Else: `new GEPA(actionabilityMetric, { numIterations, mutationsPerStep: 2, frontierStore: store })` (recommended — reviews benefit from the prompt genuinely evolving against the weakest cases). +4. `const tuned = await opt.compile(trainingProgram, reviewsSet);` → `opt.save('src/dspy/code-reviewer.gepa.json');` +5. Report `opt.result` (best meanScore; for GEPA the frontier — instruction + meanScore — and the reflections; `warmStarted`), and the delta vs the raw reviewer on a held-out slice. Flat scores ⇒ the labelled set or the metric (do your `knownIssues.near` markers actually match? do you have both `ship` and `changes`/`blocked` cases?), not the budget. For ongoing improvement across releases, this is exactly `dspy-evolution`'s territory — `/dspy-evolve` with a held-out review benchmark. diff --git a/plugins/dspy-appliance-code-review/mcp/server.js b/plugins/dspy-appliance-code-review/mcp/server.js new file mode 100644 index 0000000..56aae9c --- /dev/null +++ b/plugins/dspy-appliance-code-review/mcp/server.js @@ -0,0 +1,31 @@ +#!/usr/bin/env node +/** + * dspy-appliance-code-review MCP server — a pre-wired DSPy.ts code-review pipeline. + * Tools: code_review_init (scaffold the appliance + build the repo-context index), + * code_review_run (retrieve repo context for a diff + ChainOfThought structured review), + * code_review_tune (GEPA/MIPROv2-tune the reviewer against labelled reviews + the actionability metric), + * code_review_status (context index stats + whether a tuned reviewer is loaded). + * Resources: dspy://code-reviewer/template, dspy://review-context-indexing, dspy://review-actionability-metric. + * + * Scaffold: handlers shell out to `npx ts-node` against src/dspy/* (the copied appliance) + * and dspy.ts (RetrieveModule, ChainOfThought, Pipeline, AgentDBClient, GEPA, MIPROv2, + * CachingLM, CompilationTracer). Flesh out the @modelcontextprotocol/sdk stdio wiring + handlers. + */ +'use strict'; +const TOOLS = [ + { name: 'code_review_init', description: 'Scaffold the DSPy.ts code-review appliance into a repo: copy the program template to src/dspy/code-reviewer.ts (+ spec), build the AgentDB repo-context index (conventions, ADRs, representative modules, prior PR reviews). Returns created paths + index stats.', inputSchema: { type: 'object', properties: { dest: { type: 'string' }, contextPath: { type: 'string' }, include: { type: 'string', description: 'comma globs of repo material to index' }, priorReviews: { type: 'string', description: 'JSON of past PR reviews to index' } } } }, + { name: 'code_review_run', description: 'Review a diff or file: retrieves relevant repo context (conventions, related code, prior reviews), produces a structured review. Returns {summary, findings:[{severity,location,issue,suggestion}], questions, passages, context}.', inputSchema: { type: 'object', properties: { target: { type: 'string', description: 'path to a .diff/.patch, a source file, or "-" for git diff/stdin' }, intent: { type: 'string' }, program: { type: 'string' }, severityMin: { type: 'string', enum: ['nit', 'minor', 'major', 'blocker'] } }, required: ['target'] } }, + { name: 'code_review_tune', description: 'Tune the code-review reviewer with GEPA (or MIPROv2) against a labelled set ([{input:{diff,intent?}, output:{knownIssues:[{severity,near}], verdict}}]) and the actionability metric (coverage − noise, specificity, calibration; false alarms on clean PRs penalised). Saves .gepa.json; returns {bestScore, frontier|trials, reflections, warmStarted, delta}.', inputSchema: { type: 'object', properties: { reviewsSet: { type: 'string' }, program: { type: 'string' }, iterations: { type: 'number' }, frontierPath: { type: 'string' }, cachePath: { type: 'string' }, useMipro: { type: 'boolean' } }, required: ['reviewsSet'] } }, + { name: 'code_review_status', description: 'Code-review appliance status: repo-context AgentDB index stats (vectors, dimension, tiers, quantization) and whether a tuned reviewer (.gepa.json / .optimized.json) is present.', inputSchema: { type: 'object', properties: { contextPath: { type: 'string' }, program: { type: 'string' } } } }, +]; +const RESOURCES = [ + { uri: 'dspy://code-reviewer/template', name: 'Code-reviewer program template', description: 'The DSPy.ts code-review appliance source — RetrieveModule → ChainOfThought + actionabilityMetric.', mimeType: 'text/typescript' }, + { uri: 'dspy://review-context-indexing', name: 'Review context indexing guide', description: 'What to index (conventions/ADRs/representative code/prior reviews), metadata, tiers, retrieval tuning.', mimeType: 'text/markdown' }, + { uri: 'dspy://review-actionability-metric', name: 'Review actionability metric guide', description: 'How a review is scored, building a non-nitpicky labelled set, tuning and validating.', mimeType: 'text/markdown' }, +]; +module.exports = { TOOLS, RESOURCES }; +// TODO: wire @modelcontextprotocol/sdk StdioServerTransport; handlers shell out to +// `npx ts-node` against src/dspy/* and dspy.ts (RetrieveModule, ChainOfThought, Pipeline, AgentDBClient, GEPA, MIPROv2, CachingLM, CompilationTracer). +if (require.main === module) { + process.stderr.write('[dspy-appliance-code-review mcp] scaffold — handlers shell out to `npx ts-node` against src/dspy/* and dspy.ts. Tools: ' + TOOLS.map(t => t.name).join(', ') + '\n'); +} diff --git a/plugins/dspy-appliance-code-review/skills/review-actionability-metric/SKILL.md b/plugins/dspy-appliance-code-review/skills/review-actionability-metric/SKILL.md new file mode 100644 index 0000000..77f99af --- /dev/null +++ b/plugins/dspy-appliance-code-review/skills/review-actionability-metric/SKILL.md @@ -0,0 +1,27 @@ +--- +name: review-actionability-metric +version: "0.1.0" +author: rUv +tags: [dspy, appliance, code-review, metric, gepa] +description: > + How the DSPy.ts code-review appliance scores a review — coverage of real issues minus noise, specificity (located + concrete fix), severity calibration, no false alarms on clean PRs — and how to build a labelled review set that doesn't teach the reviewer to nitpick. + Use when: tuning the reviewer (`/code-review-tune`), building its labelled set, or validating reviews. +--- +# Scoring a code review + +The appliance ships `actionabilityMetric` ≈ `0.4 × coverage + 0.35 × specificity + 0.25 × calibration`: +- **Coverage** — against `gold.knownIssues`: did it flag the real problems (recall), without burying them in noise (penalty for findings beyond the known set)? When `gold.verdict === 'ship'`: any `blocker`/`major` finding tanks coverage (false alarm on a clean PR). +- **Specificity** — fraction of findings that have a `location` AND a concrete `suggestion`. "Consider refactoring" with no location scores ~0; "src/a.ts:12 — unhandled null on `user`; guard with `if (!user) return`" scores 1. +- **Calibration** — some spread of severities is healthy when there are findings; all-nits or all-blockers is suspect. (Zero findings on a `ship` PR is perfectly calibrated.) + +## Building the labelled set (the part people get wrong) +- From real past PRs: `{ input:{diff, intent}, output:{knownIssues:[{severity, near}], verdict:'ship'|'changes'|'blocked'} }`. `near` is a hint (file/symbol/keyword) the metric uses to check whether a finding matches a known issue. +- **Include clean PRs** — `verdict:'ship'`, empty `knownIssues`. ~25% is reasonable. Without them every example rewards finding problems, and tuning produces a reviewer that *always* finds problems. The single most important thing in this set. +- Span the kinds of issues you care about (correctness, security, convention violations, test gaps) — the tuned reviewer is only as good at catching what's represented. +- Hold out a slice; never tune on it. +- Goodhart watch: the metric can be gamed by emitting one perfectly-formatted finding per PR. Spot-check — does the tuned reviewer's output read like a senior engineer's, or like metric-bait? + +## Tuning & validating +- `/code-review-tune ` — GEPA against `actionabilityMetric` (the prompt benefits from evolving against the weakest cases), `frontierStore` + a `CachingLM`. `--mipro` for a faster, gentler pass. Read the reflections (`dspy-evolution`'s `/dspy-evolution-status --reflections` if you used `frontierStore`): which instruction change made findings more specific / cut the noise? +- Flat scores ⇒ the labelled set or the metric — do `near` markers actually match findings? both `ship` and `changes`/`blocked` cases present? leakage? — not the budget. +- Validate with `/code-review-run` on held-out diffs: recall on known issues, false-alarm rate on clean PRs, are findings located + actionable, does it cite real conventions. diff --git a/plugins/dspy-appliance-code-review/skills/review-context-indexing/SKILL.md b/plugins/dspy-appliance-code-review/skills/review-context-indexing/SKILL.md new file mode 100644 index 0000000..bde8411 --- /dev/null +++ b/plugins/dspy-appliance-code-review/skills/review-context-indexing/SKILL.md @@ -0,0 +1,34 @@ +--- +name: review-context-indexing +version: "0.1.0" +author: rUv +tags: [dspy, appliance, code-review, rag, agentdb, conventions] +description: > + How to build the AgentDB repo-context index the DSPy.ts code-review appliance retrieves from — what to index (conventions, representative code, ADRs, prior reviews), what NOT to, metadata, and tiers — so the reviewer judges against the project's real patterns. + Use when: running `/code-review-init` or extending a review context index. +--- +# Indexing repo context for code review + +The reviewer retrieves a handful of context chunks per diff and judges the change against them. Garbage in ⇒ a reviewer that invents rules or misses real ones. + +## Index this +- **Conventions / style docs** — `CONVENTIONS.md`, `CONTRIBUTING.md`, lint config rationale, the project's stated rules. `kind: 'convention'`. +- **ADRs** — architecture decisions the reviewer should hold changes to. `kind: 'adr'`. +- **A representative sample of code** — the modules that *exemplify* the patterns you want enforced (the canonical service, the canonical test, the canonical error-handling). Not the whole repo. `kind: 'code'`. +- **Prior PR reviews** — past review comments (ideally with the diff snippet they were about). This is the highest-signal source: it encodes what this team actually flags and how. `kind: 'review'`. + +## Don't index +- The entire source tree — it dilutes retrieval; a 10k-chunk dump means the relevant convention is buried under 9,990 lines of unrelated code. +- Generated code, vendored deps, lockfiles, build output. +- Stale docs / superseded ADRs — or mark them and `evictTier` later, so the reviewer doesn't enforce dead rules. + +## Mechanics +- Chunk on natural units (a doc section, a function, a review comment + its snippet), ~500–1000 chars, ~15% overlap. Keep `{ source, kind, title?, line? }` metadata — the reviewer cites the convention/related code, so it needs to know where it came from. +- `storeText(chunk, meta, { tier: 'long' })`. `promote(...)` the conventions/ADRs to `working` (they're always relevant); let one-off review snippets live in `short`/`long`. +- Big index ⇒ `quantization: 'rabitq'` + `coarseThenRerank` (`rerankFactor ≥ 3`). +- Re-index conventions/ADRs when they change (don't just append); old + new competing in retrieval = inconsistent reviews. + +## Tuning retrieval +- `k`: 5–8 — enough to bring conventions + related code + a prior review, not so much it floods the prompt. +- `mmrLambda` low-ish (~0.4) and `useMMR: true` — you want *diverse* context (a convention AND related code AND a prior review), not 6 chunks of the same convention. `overFetchFactor ≥ 3`. +- Debug misses with `dspy-rag`'s `/dspy-retrieve "\n"`. diff --git a/plugins/dspy-appliance-code-review/templates/code-reviewer.spec.ts b/plugins/dspy-appliance-code-review/templates/code-reviewer.spec.ts new file mode 100644 index 0000000..e14fc2c --- /dev/null +++ b/plugins/dspy-appliance-code-review/templates/code-reviewer.spec.ts @@ -0,0 +1,23 @@ +import { configureLM, DummyLM } from 'dspy.ts'; +import { reviewer, actionabilityMetric } from './code-reviewer'; + +describe('code-review appliance', () => { + beforeEach(async () => { const lm = new DummyLM(); await lm.init(); configureLM(lm); }); + + it('reviewer runs and returns a summary', async () => { + const out = await reviewer.run({ diff: '+ const x = 1;\n- var x = 1;', context: 'Convention: prefer const over var. (src/style.md)' }); + expect(typeof out.summary).toBe('string'); + }); + + it('actionabilityMetric rewards specific, located findings over vague ones', () => { + const specific = actionabilityMetric({ diff: 'd', context: 'c' }, { summary: 's', findings: [{ severity: 'major', location: 'src/a.ts:12', issue: 'unhandled null', suggestion: 'guard with `if (!x) return`' }] }); + const vague = actionabilityMetric({ diff: 'd', context: 'c' }, { summary: 's', findings: [{ severity: 'major', issue: 'consider refactoring', suggestion: '' }] }); + expect(specific).toBeGreaterThan(vague); + }); + + it('actionabilityMetric penalises blocker/major findings on a PR that should ship', () => { + const clean = actionabilityMetric({ diff: 'd', context: 'c' }, { summary: 'ship it', findings: [] }, { verdict: 'ship' }); + const falseAlarm = actionabilityMetric({ diff: 'd', context: 'c' }, { summary: 'blocked', findings: [{ severity: 'blocker', location: 'src/a.ts:1', issue: 'x', suggestion: 'y' }] }, { verdict: 'ship' }); + expect(clean).toBeGreaterThan(falseAlarm); + }); +}); diff --git a/plugins/dspy-appliance-code-review/templates/code-reviewer.ts b/plugins/dspy-appliance-code-review/templates/code-reviewer.ts new file mode 100644 index 0000000..7bc0106 --- /dev/null +++ b/plugins/dspy-appliance-code-review/templates/code-reviewer.ts @@ -0,0 +1,106 @@ +/** + * DSPy.ts code-review appliance — RetrieveModule pulls repo context (conventions, related + * code, prior reviews) from an AgentDB index → ChainOfThought produces a structured review. + * Copy this into your repo (e.g. src/dspy/code-reviewer.ts), point CONTEXT_PATH at an + * AgentDB index (build it with `/code-review-init`), configure a real LM, then tune + * `reviewer` with GEPA against the `actionabilityMetric` below. + * + * Requires: dspy.ts@^2.2.0 + */ +import { + RetrieveModule, ChainOfThought, Pipeline, AgentDBClient, + type Signature, type MetricFunction, type TrainingExample, +} from 'dspy.ts'; + +export const CONTEXT_PATH = process.env.REVIEW_CONTEXT_PATH ?? '.dspy/review-context'; + +/** Index of repo context: coding conventions, representative modules, ADRs, prior PR reviews. */ +export async function openReviewContext(): Promise { + const ctx = new AgentDBClient({ + vectorDimension: 384, + storage: { path: CONTEXT_PATH }, + performance: { batchEnabled: true /*, quantization: 'rabitq', rerankFactor: 3 */ }, + }); + await ctx.init(); + return ctx; +} + +export const reviewerSignature: Signature = { + inputs: [ + { name: 'diff', type: 'string', required: true, description: 'unified diff or file content under review' }, + { name: 'context', type: 'string', required: true, description: 'retrieved repo context: conventions, related code, prior reviews' }, + { name: 'intent', type: 'string', required: false, description: 'what the change is supposed to do (PR title/description)' }, + ], + outputs: [ + { name: 'summary', type: 'string', required: true, description: 'one-paragraph assessment: ship / changes-needed / blocked, and why' }, + { name: 'findings', type: 'object', required: true, description: 'array of { severity: "blocker"|"major"|"minor"|"nit", location, issue, suggestion } — each tied to a specific line/symbol with a concrete fix' }, + { name: 'questions', type: 'object', required: false, description: 'array of clarifying questions for the author' }, + ], +}; + +const reviewerPrompt = (i: { diff: string; context: string; intent?: string }) => [ + 'You are a senior code reviewer for THIS repository. Review the change below.', + 'Use the repo context to judge against the project\'s conventions and patterns — do not invent rules; cite the convention/related code when you flag something.', + 'Every finding MUST name a specific location (file:line or symbol), state the issue, and give a concrete suggested fix. No vague "consider refactoring".', + 'Tag each finding: blocker (must fix), major (should fix), minor (nice to fix), nit (style). Be honest — if it\'s good, say "ship it" with few/no findings; don\'t pad.', + '', + i.intent ? `Intent: ${i.intent}` : '', + `Repo context:\n${i.context}`, + '', + `Change under review:\n${i.diff}`, + '', + 'Review:', +].filter(Boolean).join('\n'); + +/** ChainOfThought reviewer — the module you tune with GEPA/MIPROv2. */ +export const reviewer = new ChainOfThought({ + name: 'CodeReviewer', + signature: reviewerSignature, + promptTemplate: reviewerPrompt, +}); + +/** The full appliance: Retrieve(repo context for this diff) → ChainOfThought review. `run({ diff, intent? })` → { summary, findings, questions, passages, context }. */ +export async function buildCodeReviewer(opts?: { k?: number; mmrLambda?: number }): Promise { + const ctx = await openReviewContext(); + const retrieve = new RetrieveModule({ + client: ctx, + k: opts?.k ?? 6, + useMMR: true, // diverse context: conventions + related code + prior reviews, not 6 near-dupes + mmrLambda: opts?.mmrLambda ?? 0.4, + overFetchFactor: 3, + textField: 'text', + }); + return new Pipeline([ + { module: retrieve, map: (i: { diff: string; intent?: string }) => ({ query: `${i.intent ?? ''}\n${i.diff}`.slice(0, 4000) }), merge: (i, o: { context: string }) => ({ ...i, context: o.context }) }, + { module: reviewer }, + ] as any); +} + +/** Actionability metric: a good review is specific, located, severity-calibrated, and matches the known issues — not a wall of vague nits, not silence on real problems. */ +export const actionabilityMetric: MetricFunction = ( + _in: { diff: string; context: string }, + out: { summary?: string; findings?: { severity: string; location?: string; issue?: string; suggestion?: string }[] }, + gold?: { knownIssues?: { severity: string; near?: string }[]; verdict?: 'ship' | 'changes' | 'blocked' }, +) => { + if (!out?.summary || !Array.isArray(out.findings)) return 0; + const f = out.findings; + // specificity: each finding should have a location AND a concrete suggestion + const specific = f.length === 0 ? 1 : f.filter((x) => x.location && x.suggestion && x.suggestion.length > 8).length / f.length; + // calibration: it shouldn't be all nits or all blockers; some spread is healthy when there are findings + const sev = new Set(f.map((x) => x.severity)); + const calibrated = f.length === 0 ? 1 : Math.min(1, 0.5 + 0.5 * (sev.size / 4)); + // coverage vs known issues (if we have gold): did it catch the real ones, without drowning them in noise? + let coverage = 0.6; + if (gold?.knownIssues?.length) { + const hit = gold.knownIssues.filter((ki) => f.some((x) => (x.location ?? '').includes(ki.near ?? '') || (x.issue ?? '').length > 0 && JSON.stringify(x).toLowerCase().includes((ki.near ?? '').toLowerCase()))).length; + const recall = hit / gold.knownIssues.length; + const noise = Math.max(0, f.length - gold.knownIssues.length) / Math.max(1, gold.knownIssues.length); + coverage = Math.max(0, recall - 0.15 * noise); + } else if (gold?.verdict === 'ship') { + coverage = f.filter((x) => x.severity === 'blocker' || x.severity === 'major').length === 0 ? 1 : 0.2; // false alarms on a clean PR + } + return 0.4 * coverage + 0.35 * specific + 0.25 * calibrated; +}; + +/** Tuning example shape. gold.knownIssues describes the real problems; gold.verdict the expected call. */ +export type ReviewExample = TrainingExample<{ diff: string; intent?: string }, { knownIssues?: { severity: string; near?: string }[]; verdict?: 'ship' | 'changes' | 'blocked' }>; diff --git a/plugins/dspy-appliance-data-pipeline/.claude-plugin/plugin.json b/plugins/dspy-appliance-data-pipeline/.claude-plugin/plugin.json new file mode 100644 index 0000000..d290898 --- /dev/null +++ b/plugins/dspy-appliance-data-pipeline/.claude-plugin/plugin.json @@ -0,0 +1,10 @@ +{ + "name": "dspy-appliance-data-pipeline", + "description": "Vertical appliance: a pre-wired DSPy.ts ETL / classification pipeline. Ships a ready-to-run program — typed PredictModules composed in a Pipeline (e.g. clean → classify → extract), a CSV/JSONL batch loader, a row-level metric, and BootstrapFewShot tuning from a labeled CSV. Commands scaffold it into your repo, run a batch, and tune from labels.", + "version": "0.1.0", + "author": { "name": "rUv", "url": "https://github.com/ruvnet" }, + "homepage": "https://github.com/ruvnet/dspy.ts", + "license": "MIT", + "keywords": ["dspy", "dspy.ts", "appliance", "data-pipeline", "etl", "classification", "batch", "bootstrap-fewshot", "agentdb"], + "mcpServers": { "dspy-appliance-data-pipeline": { "command": "node", "args": ["${CLAUDE_PLUGIN_ROOT}/mcp/server.js"] } } +} diff --git a/plugins/dspy-appliance-data-pipeline/AGENTS.md b/plugins/dspy-appliance-data-pipeline/AGENTS.md new file mode 100644 index 0000000..d34e191 --- /dev/null +++ b/plugins/dspy-appliance-data-pipeline/AGENTS.md @@ -0,0 +1,13 @@ +# dspy-appliance-data-pipeline — for Codex / MCP clients + +A vertical appliance: a pre-wired DSPy.ts ETL / classification pipeline. Ships a ready-to-run program (`templates/data-pipeline.ts` — typed `PredictModule` stages (clean → classify → extract) in a `Pipeline`, a CSV/JSONL batch loader, a `rowMetric`, a `BootstrapFewShot` tuning helper) plus tooling to stand it up. MCP server: `mcp/server.js`, command `node ${CLAUDE_PLUGIN_ROOT}/mcp/server.js`. + +**Tools** +- `data_pipeline_init` — copy the appliance program into `src/dspy/data-pipeline.ts` (+ spec); set the fixed label set, the raw-text field, which stages to keep. +- `data_pipeline_run` — process a batch file (CSV/JSONL): clean → classify → extract each record. Returns the output rows (one per input; errors captured per-row), the written output-CSV path, and a summary (rows, errors, label distribution, mean confidence). +- `data_pipeline_tune` — `BootstrapFewShot`-tune the `classify` stage from a labeled file + the row metric (labeled + self-bootstrapped demos; optional input-conditioned dynamic demos via AgentDB); saves `.optimized.json`; returns held-out accuracy raw vs tuned + a confusion matrix. +- `data_pipeline_eval` — evaluate the classify stage (raw / optimized / both): accuracy, per-class precision/recall, confusion matrix, label distribution. + +**Resources**: `dspy://data-pipeline/template`, `dspy://typed-pipeline-design`, `dspy://batch-classification-metrics`. + +Handlers shell out to `npx ts-node` against `src/dspy/*` (the copied appliance) and `dspy.ts`. Slash commands `/data-pipeline-init`, `/data-pipeline-run`, `/data-pipeline-tune` wrap the same flows; the `data-pipeline-builder` agent designs the stages, wires batch I/O, defines the metric, and tunes from labels. Builds on `dspy-optimize` (BootstrapFewShot / MIPROv2 / `/dspy-evolve` for stronger tuning) and `dspy-observability` (`CachingLM` for big batches). diff --git a/plugins/dspy-appliance-data-pipeline/agents/data-pipeline-builder.md b/plugins/dspy-appliance-data-pipeline/agents/data-pipeline-builder.md new file mode 100644 index 0000000..94a56bc --- /dev/null +++ b/plugins/dspy-appliance-data-pipeline/agents/data-pipeline-builder.md @@ -0,0 +1,15 @@ +--- +name: data-pipeline-builder +description: Builds and tunes the DSPy.ts data-pipeline appliance for a specific schema — designs the typed PredictModule stages and their Pipeline order, wires the CSV/JSONL batch I/O, defines the row metric, BootstrapFewShot-tunes the classify stage from a labeled sample, evaluates held-out accuracy, and reports. Use to turn "process this batch of records: clean them, classify into , extract " into a tuned, runnable pipeline. +--- +You build data-pipeline appliances on DSPy.ts. + +STEPS: +1. **Scaffold** — `/data-pipeline-init --labels --raw-field --stages `: copies the appliance (typed `PredictModule` stages in a `Pipeline` + a CSV/JSONL loader + `rowMetric` + a `BootstrapFewShot` helper). Adjust each stage's `Signature` to the real schema — semantic field names, one output per thing you'll score, types validated at runtime; keep stages small (a fat 12-field signature optimizes badly — split it). +2. **Pipeline order** — clean before classify before extract; later stages `merge` the earlier outputs into their input. Drop stages you don't need (pure classification = just `classify`); add ones you do (e.g. a `route` stage, a `redact` stage). Each stage is a separate `PredictModule` so you can tune them independently. +3. **Batch I/O** — `parseDelimited` (CSV/JSONL) → `runBatch` (one output row per input, errors captured per-row as `_error`) → `toCSV`. Wire your real source/sink if it's not flat files. +4. **Row metric** — `rowMetric` scores the `classify` stage: exact label = 1, off-vocabulary = capped, real-class↔"other" confusion penalised, partial credit for a wrong-but-real attempt. Graded, so BootstrapFewShot/MIPROv2 have a gradient. Adjust to your cost model (some confusions are worse than others). (See the `batch-classification-metrics` skill.) +5. **Tune** — label a stratified sample (cover every class), `/data-pipeline-tune ` — `BootstrapFewShot` (labeled + self-bootstrapped demos; add `dynamicDemos` for heterogeneous inputs) against `rowMetric`, held-out evaluation + confusion matrix. Stronger pass: `dspy-optimize`'s `/dspy-mipro` (instructions + demos) on `classify`; whole-pipeline prompt evolution: `/dspy-evolve`. Big batches: wrap the LM in `CachingLM`. +6. **Validate** — `/data-pipeline-run --use-optimized` on a held-out batch; check accuracy, the label distribution (is "other" dominating? a class missing?), confidence calibration, error rate. + +DELIVER: the program file, the batch I/O wiring, the row metric, the tuned `classify` (`.optimized.json`), and a report (held-out accuracy raw vs tuned, confusion matrix, label distribution, error rate). diff --git a/plugins/dspy-appliance-data-pipeline/commands/data-pipeline-init.md b/plugins/dspy-appliance-data-pipeline/commands/data-pipeline-init.md new file mode 100644 index 0000000..a2cf950 --- /dev/null +++ b/plugins/dspy-appliance-data-pipeline/commands/data-pipeline-init.md @@ -0,0 +1,10 @@ +--- +description: Scaffold the DSPy.ts data-pipeline appliance into your repo — copy the program template to src/dspy/data-pipeline.ts (+ spec), with typed PredictModules in a Pipeline (clean → classify → extract), a CSV/JSONL loader, a row metric, and a BootstrapFewShot tuning helper. +argument-hint: "[--dest src/dspy/data-pipeline.ts] [--labels billing,bug,feature_request,how_to,other] [--raw-field raw] [--stages clean,classify,extract]" +--- +Set up the data-pipeline appliance. Parse `$ARGUMENTS` for `--dest` (default `src/dspy/data-pipeline.ts`), `--labels` (the fixed classification label set; rewrite `LABELS` in the template), `--raw-field` (the input column name, default `raw`), `--stages` (which stages to keep — default all three; drop `extract` for pure classification, add your own). + +1. Confirm `dspy.ts` is a dependency. +2. Copy `${CLAUDE_PLUGIN_ROOT}/templates/data-pipeline.ts` → `--dest` (+ `data-pipeline.spec.ts`). Rewrite `LABELS`, `--raw-field`, and trim/extend the `Pipeline` stages per `--stages`. +3. Adjust each stage's `Signature` to your schema (input/output field names + types — keep them semantic; one output per thing you'll score), and its `promptTemplate`. +4. Print next steps: prepare a batch file (`data.csv` with a `--raw-field` column, or `.jsonl`); `/data-pipeline-run data.csv` to process it; for tuning, label a sample (`label` column = the gold class) → `/data-pipeline-tune labeled.csv`. diff --git a/plugins/dspy-appliance-data-pipeline/commands/data-pipeline-run.md b/plugins/dspy-appliance-data-pipeline/commands/data-pipeline-run.md new file mode 100644 index 0000000..5b69da2 --- /dev/null +++ b/plugins/dspy-appliance-data-pipeline/commands/data-pipeline-run.md @@ -0,0 +1,11 @@ +--- +description: Run the data-pipeline appliance over a batch file (CSV/JSONL) — clean → classify → extract each record — and write the results out as CSV (one output row per input, errors captured per-row), with a summary. +argument-hint: " [--program src/dspy/data-pipeline.ts] [--raw-field raw] [--out out.csv] [--use-optimized]" +--- +Process a batch. Parse `$ARGUMENTS` for the input file (CSV with a raw-text column, or JSONL of objects), `--program` (default `src/dspy/data-pipeline.ts`), `--raw-field` (default `raw`), `--out` (output CSV path; default `.out.csv`), `--use-optimized` (load the tuned `classify` stage from `.optimized.json` if present). + +1. `const rows = parseDelimited(content, kind);` (from the appliance module). +2. Build the pipeline (`buildDataPipeline()`); if `--use-optimized`, swap in the tuned `classify` (`BootstrapFewShot.load(...)`). +3. `const out = await runBatch(rows, rawField);` — each output row = the input fields + `{ text, label, confidence?, fields }`, or `{ ..., _error }` if a row threw. +4. Write `toCSV(out)` to `--out`. Print a summary: rows processed, errors, the label distribution (counts per class), mean `confidence`, a few sample rows. Flag if "other" dominates (likely the label set is wrong or the prompt is weak) or if confidence is uniformly low. +5. For large batches, consider wrapping the LM in `CachingLM` (`dspy-observability`'s `/dspy-cache`) — many records produce near-identical prompts after the `clean` stage. diff --git a/plugins/dspy-appliance-data-pipeline/commands/data-pipeline-tune.md b/plugins/dspy-appliance-data-pipeline/commands/data-pipeline-tune.md new file mode 100644 index 0000000..6b6ddcf --- /dev/null +++ b/plugins/dspy-appliance-data-pipeline/commands/data-pipeline-tune.md @@ -0,0 +1,11 @@ +--- +description: Tune the data-pipeline's classify stage with BootstrapFewShot from a labeled CSV/JSONL and the row metric — labeled + self-bootstrapped demos, optionally input-conditioned via an AgentDB vector store — then report held-out accuracy. +argument-hint: " [--program src/dspy/data-pipeline.ts] [--text-field text] [--label-field label] [--labeled N] [--dynamic .dspy/data-demos] [--holdout 0.2]" +--- +Tune the `classify` stage in `src/dspy/data-pipeline.ts`. Parse `$ARGUMENTS` for the labeled file (rows with a text column and a gold label column), `--program`, `--text-field` (default `text` — if your file has only `raw`, run the `clean` stage first to produce `text`), `--label-field` (default `label`), `--labeled` (max labeled demos, default 8), `--dynamic` (AgentDB path → input-conditioned demo selection), `--holdout` (fraction held out for evaluation, default 0.2). + +1. Load the file; build `trainset: [{ input:{text}, output:{label} }]`. Split off a held-out slice (`--holdout`). +2. `const tuned = await tuneClassify(trainset, { maxLabeledDemos, dynamicStorePath });` (from the appliance module) — `BootstrapFewShot(rowMetric, { maxLabeledDemos, maxBootstrappedDemos: 4, dynamicDemos })` → a `BootstrapOptimizedModule`. +3. Save it: `opt.save('src/dspy/data-pipeline.optimized.json')` (the helper exposes the optimizer; or re-run `BootstrapFewShot` directly to get a handle). Then `/data-pipeline-run --use-optimized` swaps it in. +4. Evaluate: run the raw `classify` and the tuned one over the held-out slice with `rowMetric`; report accuracy each + the delta + a confusion matrix (which classes get mixed up). If `--dynamic`, note that `tuned.selectDemos(input)` picks the k-nearest demos per row at run time. +5. Flat / no improvement ⇒ usually the *labels* (overlapping classes — split or merge them), the *label balance* (one class dominates — stratify), or *not enough labeled demos*. For a stronger pass (instructions + demos, not just demos), `dspy-optimize`'s `/dspy-mipro` on the `classify` stage; for the whole pipeline's prompts, `/dspy-evolve`. diff --git a/plugins/dspy-appliance-data-pipeline/mcp/server.js b/plugins/dspy-appliance-data-pipeline/mcp/server.js new file mode 100644 index 0000000..99d3e79 --- /dev/null +++ b/plugins/dspy-appliance-data-pipeline/mcp/server.js @@ -0,0 +1,32 @@ +#!/usr/bin/env node +/** + * dspy-appliance-data-pipeline MCP server — a pre-wired DSPy.ts ETL / classification pipeline. + * Tools: data_pipeline_init (scaffold the appliance: typed PredictModule stages in a Pipeline + + * CSV/JSONL I/O + row metric + BootstrapFewShot helper), data_pipeline_run (process a batch + * file → output CSV, errors per-row), data_pipeline_tune (BootstrapFewShot the classify stage + * from a labeled file + the row metric), data_pipeline_eval (held-out accuracy + confusion matrix + * for raw vs tuned). + * Resources: dspy://data-pipeline/template, dspy://typed-pipeline-design, dspy://batch-classification-metrics. + * + * Scaffold: handlers shell out to `npx ts-node` against src/dspy/* (the copied appliance) + * and dspy.ts (PredictModule, Pipeline, BootstrapFewShot, AgentDBClient, CachingLM, MIPROv2). + * Flesh out the @modelcontextprotocol/sdk stdio wiring + handlers. + */ +'use strict'; +const TOOLS = [ + { name: 'data_pipeline_init', description: 'Scaffold the DSPy.ts data-pipeline appliance into a repo: copy the program template to src/dspy/data-pipeline.ts (+ spec) — typed PredictModule stages (clean → classify → extract) in a Pipeline, CSV/JSONL loader, a row metric, a BootstrapFewShot tuning helper. Returns created paths.', inputSchema: { type: 'object', properties: { dest: { type: 'string' }, labels: { type: 'array', items: { type: 'string' }, description: 'the fixed classification label set' }, rawField: { type: 'string' }, stages: { type: 'array', items: { type: 'string', enum: ['clean', 'classify', 'extract'] } } } } }, + { name: 'data_pipeline_run', description: 'Run the data pipeline over a batch file (CSV/JSONL): clean → classify → extract each record. Returns the output rows (one per input; errors captured per-row as {_error}), the path of the written output CSV, and a summary (rows, errors, label distribution, mean confidence).', inputSchema: { type: 'object', properties: { input: { type: 'string', description: 'CSV with a raw-text column, or JSONL of objects' }, program: { type: 'string' }, rawField: { type: 'string' }, out: { type: 'string' }, useOptimized: { type: 'boolean' } }, required: ['input'] } }, + { name: 'data_pipeline_tune', description: 'BootstrapFewShot-tune the classify stage from a labeled file (rows with a text column + a gold label column) and the row metric (labeled + self-bootstrapped demos, optional input-conditioned dynamic demos via AgentDB). Saves .optimized.json; returns {heldOutAccuracyRaw, heldOutAccuracyTuned, delta, confusionMatrix}.', inputSchema: { type: 'object', properties: { labeled: { type: 'string' }, program: { type: 'string' }, textField: { type: 'string' }, labelField: { type: 'string' }, maxLabeledDemos: { type: 'number' }, dynamicStorePath: { type: 'string' }, holdout: { type: 'number' } }, required: ['labeled'] } }, + { name: 'data_pipeline_eval', description: 'Evaluate the classify stage (raw and/or tuned) over a labeled dataset with the row metric: accuracy, per-class precision/recall, confusion matrix, label distribution.', inputSchema: { type: 'object', properties: { labeled: { type: 'string' }, program: { type: 'string' }, variant: { type: 'string', enum: ['raw', 'optimized', 'both'] } }, required: ['labeled'] } }, +]; +const RESOURCES = [ + { uri: 'dspy://data-pipeline/template', name: 'Data-pipeline program template', description: 'The DSPy.ts data-pipeline appliance source — typed PredictModule stages in a Pipeline + CSV/JSONL I/O + rowMetric + BootstrapFewShot helper.', mimeType: 'text/typescript' }, + { uri: 'dspy://typed-pipeline-design', name: 'Typed pipeline design guide', description: 'Stage boundaries, signature shape, the merge pattern, batch I/O, tunability.', mimeType: 'text/markdown' }, + { uri: 'dspy://batch-classification-metrics', name: 'Batch classification metrics guide', description: 'Row metric design, off-vocab/"other" handling, building a stratified labeled set, reading a confusion matrix.', mimeType: 'text/markdown' }, +]; +module.exports = { TOOLS, RESOURCES }; +// TODO: wire @modelcontextprotocol/sdk StdioServerTransport; handlers shell out to +// `npx ts-node` against src/dspy/* and dspy.ts (PredictModule, Pipeline, BootstrapFewShot, AgentDBClient, CachingLM, MIPROv2). +if (require.main === module) { + process.stderr.write('[dspy-appliance-data-pipeline mcp] scaffold — handlers shell out to `npx ts-node` against src/dspy/* and dspy.ts. Tools: ' + TOOLS.map(t => t.name).join(', ') + '\n'); +} diff --git a/plugins/dspy-appliance-data-pipeline/skills/batch-classification-metrics/SKILL.md b/plugins/dspy-appliance-data-pipeline/skills/batch-classification-metrics/SKILL.md new file mode 100644 index 0000000..de37dbe --- /dev/null +++ b/plugins/dspy-appliance-data-pipeline/skills/batch-classification-metrics/SKILL.md @@ -0,0 +1,32 @@ +--- +name: batch-classification-metrics +version: "0.1.0" +author: rUv +tags: [dspy, appliance, data-pipeline, metric, classification, evaluation] +description: > + How to design the row-level metric for a DSPy.ts data pipeline and read its batch evaluation — graded accuracy, off-vocabulary/"other" handling, confusion matrices, label balance — so BootstrapFewShot/MIPROv2 have a gradient and you can see what's actually wrong. + Use when: tuning the classify stage (`/data-pipeline-tune`), building the labeled set, or reading a batch run. +--- +# Row metrics & batch evaluation + +The appliance's `rowMetric` scores the `classify` stage per record, in [0,1]: exact gold label = 1; off-vocabulary label = ~0.1; confusing a real class with `"other"` (either direction) = ~0.2; wrong-but-a-real-class = ~0.3; no gold = ~0.5 (a confident specific label) / ~0.3 (`"other"`). + +## Designing the metric +- **Graded, not binary.** Exact-match-only gives the optimizer no gradient — most candidates score 0 and BootstrapFewShot/MIPROv2 wander. Partial credit for a real attempt is what lets the search climb. (See `dspy-core`'s `metric-design`.) +- **Penalise off-vocabulary hard.** A label not in your fixed set is broken output; cap it near 0 so the optimizer learns to stay in vocabulary. +- **Encode your cost model.** Not all confusions are equal — if mislabeling `billing` as `bug` is far worse than `bug` as `feature_request`, weight the metric (e.g. a small confusion-cost table) so tuning optimizes what you actually care about. +- **`"other"` is a trap.** A model under pressure dumps everything into `"other"`. Penalise real↔`"other"` confusion in both directions, and watch the output distribution. +- **Multi-output stages** — if `classify` also outputs `confidence`, you can add a calibration term (reward confidence that tracks correctness), but keep it secondary; accuracy first. + +## Building the labeled set +- **Stratify** — cover every class, roughly proportional to reality (or oversample rare classes if you care about recall on them). A set that's 90% one class teaches the model to always guess that class. +- Real examples, not synthetic — the model needs to see the actual phrasing it'll face. +- Hold out a slice (`--holdout`); never tune on it. +- Goodhart watch — if the metric over-weights one class, the tuned classifier will over-predict it. Spot-check the confusion matrix. + +## Reading a batch evaluation +`/data-pipeline-tune` reports raw vs tuned accuracy on the held-out slice, the delta, and a **confusion matrix** — read it: +- A whole row/column dominated by `"other"` ⇒ the label set is wrong (a class is missing, or two should be merged) or the prompt is weak. +- Two classes that swap a lot ⇒ they overlap; sharpen the signature `description` (what distinguishes them) or merge them. +- Flat delta after tuning ⇒ labels/balance/not-enough-demos, not budget. Try `dspy-optimize`'s `/dspy-mipro` (instructions + demos) before throwing more data at it. +- `/data-pipeline-run` also prints the live label distribution + mean confidence — a fast smell test on unlabeled batches. diff --git a/plugins/dspy-appliance-data-pipeline/skills/typed-pipeline-design/SKILL.md b/plugins/dspy-appliance-data-pipeline/skills/typed-pipeline-design/SKILL.md new file mode 100644 index 0000000..155df90 --- /dev/null +++ b/plugins/dspy-appliance-data-pipeline/skills/typed-pipeline-design/SKILL.md @@ -0,0 +1,31 @@ +--- +name: typed-pipeline-design +version: "0.1.0" +author: rUv +tags: [dspy, appliance, data-pipeline, pipeline, predict, etl] +description: > + How to design a DSPy.ts data pipeline as typed PredictModule stages composed in a Pipeline — stage boundaries, signature shape, the merge pattern, error handling, and batch I/O — so each stage is tunable and the batch is robust. + Use when: building/refactoring the data-pipeline appliance (`/data-pipeline-init`, `/data-pipeline-run`). +--- +# Designing a typed DSPy.ts pipeline + +A `Pipeline` runs `PredictModule` stages in order; each stage's input is built from the running record, its output `merge`d back in. The appliance ships `clean → classify → extract`. + +## Stage boundaries +- **One job per stage.** `clean` normalizes, `classify` picks a label, `extract` pulls fields. Don't make one module do all three — a fat signature optimizes badly and a single bad output drags the whole thing down. If a stage's signature has >~6 fields, it's probably two stages. +- **Order by dependency.** `clean` before `classify` (classify the cleaned text), `classify` before `extract` (extraction depends on the label). Later stages `merge` earlier outputs into their input (`merge: (i, o) => ({ ...i, ...o })`). +- **Add/remove stages freely.** Pure classification? Just `classify`. Need routing? A `route` stage. Need redaction? A `redact` stage early. Each is a separate `PredictModule` you can tune independently. + +## Signature shape +- Semantic field names — `text`, `label`, `confidence`, `fields` — not `out1`. The names go into the prompt. +- One output per thing you'll score. If the metric checks the label *and* a confidence, make them two outputs. +- Types are validated at runtime (`Module.validateInput/validateOutput`). `object` covers arrays/maps. A `description` on a field constrains the model (`"one of: a, b, c"`, `"0..1"`). +- A fixed enum (like `LABELS`) belongs in the signature's `description` *and* in the metric (penalise off-vocabulary outputs). + +## Batch I/O & robustness +- `parseDelimited(content, 'csv'|'jsonl')` → rows; `runBatch(rows, rawField)` → one output row per input, **errors captured per-row** (`{ ..., _error }`) so one bad record doesn't kill the batch; `toCSV(rows)` out. Wire your real source/sink if not flat files. +- For large batches, wrap the LM in `CachingLM` (`dspy-observability`'s `/dspy-cache`) — after `clean`, many records produce near-identical prompts; the cache turns the run from N calls into ~(distinct prompts). +- Watch the output distribution: if `"other"` dominates, the label set or the `classify` prompt is wrong; if `confidence` is uniformly low, the prompt isn't giving the model enough to decide on. + +## Tunability +Because each stage is its own `PredictModule`, you can: `BootstrapFewShot` the `classify` stage from labels (`/data-pipeline-tune`), `MIPROv2` it for instructions + demos (`dspy-optimize`'s `/dspy-mipro`), or evolve the whole pipeline's prompts (`dspy-evolution`'s `/dspy-evolve`). Tune the stage that's the bottleneck — check per-stage error rates first. diff --git a/plugins/dspy-appliance-data-pipeline/templates/data-pipeline.spec.ts b/plugins/dspy-appliance-data-pipeline/templates/data-pipeline.spec.ts new file mode 100644 index 0000000..3c2ef1b --- /dev/null +++ b/plugins/dspy-appliance-data-pipeline/templates/data-pipeline.spec.ts @@ -0,0 +1,26 @@ +import { configureLM, DummyLM } from 'dspy.ts'; +import { classify, rowMetric, parseDelimited, toCSV, LABELS } from './data-pipeline'; + +describe('data-pipeline appliance', () => { + beforeEach(async () => { const lm = new DummyLM(); await lm.init(); configureLM(lm); }); + + it('classify runs and returns a label', async () => { + const out = await classify.run({ text: 'my invoice is wrong' }); + expect(typeof out.label).toBe('string'); + }); + + it('rowMetric: exact label = 1, real-class↔other confusion penalised, off-vocab capped', () => { + expect(rowMetric({ text: 't' }, { label: 'billing' }, { label: 'billing' })).toBe(1); + expect(rowMetric({ text: 't' }, { label: 'other' }, { label: 'billing' })).toBeLessThan(rowMetric({ text: 't' }, { label: 'bug' }, { label: 'billing' })); + expect(rowMetric({ text: 't' }, { label: 'nonsense' }, { label: 'billing' })).toBeLessThanOrEqual(0.1); + }); + + it('CSV round-trips', () => { + const rows = parseDelimited('raw,label\nhello,how_to\n"a, b",bug'); + expect(rows).toHaveLength(2); + expect(rows[1].raw).toBe('a, b'); + expect(toCSV(rows)).toContain('raw,label'); + }); + + it('LABELS is a fixed non-empty set', () => { expect(LABELS.length).toBeGreaterThan(1); }); +}); diff --git a/plugins/dspy-appliance-data-pipeline/templates/data-pipeline.ts b/plugins/dspy-appliance-data-pipeline/templates/data-pipeline.ts new file mode 100644 index 0000000..90b3df9 --- /dev/null +++ b/plugins/dspy-appliance-data-pipeline/templates/data-pipeline.ts @@ -0,0 +1,115 @@ +/** + * DSPy.ts data-pipeline appliance — typed PredictModules composed in a Pipeline that processes + * a batch of records (clean → classify → extract), tuned with BootstrapFewShot from a labeled CSV. + * Copy this into your repo (e.g. src/dspy/data-pipeline.ts), adjust the signatures/stages to your + * schema, configure a real LM, then tune with BootstrapFewShot against `rowMetric`. + * + * Requires: dspy.ts@^2.2.0 + */ +import { + PredictModule, Pipeline, AgentDBClient, + BootstrapFewShot, type BootstrapOptimizedModule, + type Signature, type MetricFunction, type TrainingExample, +} from 'dspy.ts'; + +/** ---- Stage 1: normalize the raw text ---- */ +const cleanSig: Signature = { + inputs: [{ name: 'raw', type: 'string', required: true }], + outputs: [{ name: 'text', type: 'string', required: true, description: 'the input with boilerplate/markup stripped, whitespace normalized' }], +}; +export const clean = new PredictModule<{ raw: string }, { text: string }>({ + name: 'Clean', signature: cleanSig, + promptTemplate: (i) => `Strip boilerplate/markup and normalize whitespace. Return only the cleaned text.\n---\n${i.raw}`, +}); + +/** ---- Stage 2: classify into a fixed label set ---- */ +export const LABELS = ['billing', 'bug', 'feature_request', 'how_to', 'other'] as const; +const classifySig: Signature = { + inputs: [{ name: 'text', type: 'string', required: true }], + outputs: [ + { name: 'label', type: 'string', required: true, description: `one of: ${LABELS.join(', ')}` }, + { name: 'confidence', type: 'number', required: false, description: '0..1' }, + ], +}; +export const classify = new PredictModule<{ text: string }, { label: string; confidence?: number }>({ + name: 'Classify', signature: classifySig, + promptTemplate: (i) => `Classify the message into exactly one of: ${LABELS.join(', ')}.\nReturn the label (and a 0..1 confidence).\n---\n${i.text}`, +}); + +/** ---- Stage 3: extract structured fields (only meaningful for some labels) ---- */ +const extractSig: Signature = { + inputs: [{ name: 'text', type: 'string', required: true }, { name: 'label', type: 'string', required: true }], + outputs: [{ name: 'fields', type: 'object', required: true, description: 'extracted fields relevant to the label; {} if none apply' }], +}; +export const extract = new PredictModule<{ text: string; label: string }, { fields: Record }>({ + name: 'Extract', signature: extractSig, + promptTemplate: (i) => `For a "${i.label}" message, extract the relevant structured fields as JSON (e.g. order_id, version, feature). Return {} if none apply.\n---\n${i.text}`, +}); + +/** The full pipeline: clean → classify → extract. `run({ raw })` → { text, label, confidence?, fields }. */ +export function buildDataPipeline(): Pipeline { + return new Pipeline([ + { module: clean }, + { module: classify, merge: (i: { text: string }, o: { label: string; confidence?: number }) => ({ ...i, ...o }) }, + { module: extract, merge: (i, o: { fields: Record }) => ({ ...i, ...o }) }, + ] as any); +} + +/** ---- batch I/O (CSV / JSONL) ---- */ +export type Row = Record; +export function parseDelimited(content: string, kind: 'csv' | 'jsonl' = 'csv'): Row[] { + if (kind === 'jsonl') return content.split(/\r?\n/).filter(Boolean).map((l) => JSON.parse(l)); + const [head, ...lines] = content.split(/\r?\n/).filter((l) => l.length); + const cols = head.split(',').map((c) => c.trim()); + return lines.map((l) => { const v = l.split(','); return Object.fromEntries(cols.map((c, k) => [c, (v[k] ?? '').trim()])); }); +} +export function toCSV(rows: Record[]): string { + if (!rows.length) return ''; + const cols = Array.from(rows.reduce((s, r) => { Object.keys(r).forEach((k) => s.add(k)); return s; }, new Set())); + const esc = (v: unknown) => { const s = typeof v === 'object' ? JSON.stringify(v) : String(v ?? ''); return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s; }; + return [cols.join(','), ...rows.map((r) => cols.map((c) => esc((r as Row)[c])).join(','))].join('\n'); +} + +/** Run the pipeline over a batch. Returns one output row per input row (errors captured as { _error }). */ +export async function runBatch(rows: Row[], rawField = 'raw'): Promise[]> { + const pipe = buildDataPipeline(); + const out: Record[] = []; + for (const r of rows) { + try { out.push({ ...r, ...(await pipe.run({ raw: r[rawField] ?? '' })) }); } + catch (e) { out.push({ ...r, _error: (e as Error).message }); } + } + return out; +} + +/** Row-level metric for tuning the `classify` stage: accuracy with a partial-credit fallback; "other" misuse penalised. */ +export const rowMetric: MetricFunction = ( + _in: { text: string }, + out: { label?: string; confidence?: number }, + gold?: { label?: string }, +) => { + if (!out?.label) return 0; + const got = out.label.trim().toLowerCase(); + if (!(LABELS as readonly string[]).includes(got)) return 0.1; // off-vocabulary label + if (!gold?.label) return got === 'other' ? 0.3 : 0.5; // no gold: a confident specific label is plausibly fine + const g = gold.label.trim().toLowerCase(); + if (got === g) return 1; + if (got === 'other' || g === 'other') return 0.2; // confusing a real class with "other" (either direction) is bad + return 0.3; // wrong, but a real attempt +}; + +/** Helper: BootstrapFewShot-tune the `classify` stage from a labeled trainset (optionally with input-conditioned dynamic demos via AgentDB). */ +export async function tuneClassify( + trainset: TrainingExample<{ text: string }, { label: string }>[], + opts?: { maxLabeledDemos?: number; dynamicStorePath?: string }, +): Promise> { + let store: AgentDBClient | undefined; + if (opts?.dynamicStorePath) { store = new AgentDBClient({ vectorDimension: 384, storage: { path: opts.dynamicStorePath } }); await store.init(); } + const opt = new BootstrapFewShot(rowMetric, { + maxLabeledDemos: opts?.maxLabeledDemos ?? 8, + maxBootstrappedDemos: 4, + dynamicDemos: store ? { store, k: 3 } : undefined, + }); + return (await opt.compile(classify as any, trainset as any)) as any; +} + +export type DataExample = TrainingExample<{ text: string }, { label: string }>; diff --git a/plugins/dspy-appliance-research-assistant/.claude-plugin/plugin.json b/plugins/dspy-appliance-research-assistant/.claude-plugin/plugin.json new file mode 100644 index 0000000..394bb3c --- /dev/null +++ b/plugins/dspy-appliance-research-assistant/.claude-plugin/plugin.json @@ -0,0 +1,10 @@ +{ + "name": "dspy-appliance-research-assistant", + "description": "Vertical appliance: a pre-wired DSPy.ts research assistant. Ships a ready-to-run program — a ReAct agent over a search/fetch/notes tool registry, with ReActReflexion (recalls research lessons, records episodes, promotes successful search strategies into skills) → a ChainOfThought synthesizer that writes a grounded, cited answer → a groundedness/coverage metric → MIPROv2 tuning. Commands scaffold it into your repo, run a research query, and tune it.", + "version": "0.1.0", + "author": { "name": "rUv", "url": "https://github.com/ruvnet" }, + "homepage": "https://github.com/ruvnet/dspy.ts", + "license": "MIT", + "keywords": ["dspy", "dspy.ts", "appliance", "research-assistant", "react", "reflexion", "synthesis", "citations", "agentdb"], + "mcpServers": { "dspy-appliance-research-assistant": { "command": "node", "args": ["${CLAUDE_PLUGIN_ROOT}/mcp/server.js"] } } +} diff --git a/plugins/dspy-appliance-research-assistant/AGENTS.md b/plugins/dspy-appliance-research-assistant/AGENTS.md new file mode 100644 index 0000000..64bd47c --- /dev/null +++ b/plugins/dspy-appliance-research-assistant/AGENTS.md @@ -0,0 +1,13 @@ +# dspy-appliance-research-assistant — for Codex / MCP clients + +A vertical appliance: a pre-wired DSPy.ts research assistant. Ships a ready-to-run program (`templates/research-assistant.ts` — a `ReAct` agent over a search/fetch/note tool registry with `ReActReflexion` → a `ChainOfThought` synthesizer that writes a grounded, cited answer + a `groundedAnswerMetric`) plus tooling to stand it up. MCP server: `mcp/server.js`, command `node ${CLAUDE_PLUGIN_ROOT}/mcp/server.js`. + +**Tools** +- `research_init` — copy the appliance program into `src/dspy/research-assistant.ts` (+ spec), create the AgentDB reflexion store. The `search`/`fetch` tools are stubs — wire them to a real backend (web search API, your corpus, an AgentDB retriever); the `note` tool is the bridge to the synthesizer. +- `research_ask` — run a research query: ReAct gathers evidence (search → fetch → note), ChainOfThought synthesizes. Returns `{answer, citations:[{source,claim}], gaps, steps, evidence, recalledLessons, promotedSkill}`. +- `research_tune` — MIPROv2-tune the `synthesizer` (cheap) or `gatherer` (ReAct thought prompt) or `both` against a graded set (`[{input:{question}, output:{mustCover:[...], answerable}}]` — include `answerable:false` cases) and the groundedness/coverage metric; AgentDB replay + tracer; saves `.optimized.json`. +- `research_status` — reflexion store stats + whether a tuned synthesizer is loaded. + +**Resources**: `dspy://research-assistant/template`, `dspy://research-tool-registry`, `dspy://synthesis-and-grounding`. + +Handlers shell out to `npx ts-node` against `src/dspy/*` (the copied appliance) and `dspy.ts`. Slash commands `/research-init`, `/research-ask`, `/research-tune` wrap the same flows; the `research-assistant-builder` agent stands up and tunes an assistant for a specific domain/backend. Builds on `dspy-react` (ReAct + reflexion), `dspy-rag` (a retriever backend for the tools), `dspy-optimize` + `dspy-evolution` (tuning), `dspy-observability` (tracer, cache). diff --git a/plugins/dspy-appliance-research-assistant/agents/research-assistant-builder.md b/plugins/dspy-appliance-research-assistant/agents/research-assistant-builder.md new file mode 100644 index 0000000..23983bd --- /dev/null +++ b/plugins/dspy-appliance-research-assistant/agents/research-assistant-builder.md @@ -0,0 +1,14 @@ +--- +name: research-assistant-builder +description: Stands up and tunes the DSPy.ts research-assistant appliance for a specific domain/backend — scaffolds the program, wires the search/fetch/note tool registry to a real backend, sets the ReAct step budget and reflexion store, builds a graded research set, MIPROv2-tunes the synthesizer (and ReAct thought prompt) against the groundedness/coverage metric, and validates answer quality. Use to go from "an assistant that researches questions over and writes cited answers" to a tuned, self-improving program. +--- +You build research-assistant appliances on DSPy.ts. + +STEPS: +1. **Scaffold** — `/research-init --reflexion `: copies the appliance (`ReAct(search/fetch/note, reflexion) → ChainOfThought(synthesize)` + `groundedAnswerMetric`) into `src/dspy/research-assistant.ts`. Keep the prompts' hard constraints — *gather with `note(source,text)` before citing; synthesize using only the gathered evidence; cite every claim; name the gaps; don't overclaim on thin/conflicting evidence*. +2. **Wire the tools** — the `search`/`fetch` handlers are stubs. Point them at the real backend: a web search API, your document corpus, an `AgentDBClient` retriever (`dspy-rag`), an internal API. Make handlers robust — errors become observations the agent must recover from; return terse, parseable strings, not 5KB blobs. The `note` tool is non-negotiable: the synthesizer only sees what was noted. (See the `research-tool-registry` skill.) +3. **Budget & reflexion** — `maxSteps` 6–10 (research needs a few search→fetch→note rounds). `ReActReflexion({ store, recallK, skillThreshold })` on a persistent path — it recalls research *lessons* (what went wrong last time) and promotes *search strategies* (sequences that kept working) into skills. Use a stable `taskKey` per research-task *type* so lessons generalise. +4. **Graded set & tune** — collect questions with `mustCover` sub-topics and a `verdict`; **include `answerable:false` cases** (questions the sources can't answer) so tuning rewards honest "the evidence is insufficient" over confident fabrication. `/research-tune ` — MIPROv2 against `groundedAnswerMetric` (grounded + covers + calibrated), with `replayStore` + a `CachingLM`. Tune `synthesizer` first (cheap), then `gatherer` if gathering is the weak link. For ongoing evolution, `dspy-evolution`'s `/dspy-evolve`. +5. **Validate** — `/research-ask` on held-out questions: are claims backed by gathered evidence (real `source` ids)? does it cover the sub-topics? does it name gaps? does it hedge when the evidence is thin? (See the `synthesis-and-grounding` skill.) + +DELIVER: the program file, the wired tool registry, the reflexion store path, the graded set, the tuned synthesizer (`.optimized.json`), and a quality report (mean metric, examples of grounded answers, hallucinated-citation rate, coverage on held-out questions). diff --git a/plugins/dspy-appliance-research-assistant/commands/research-ask.md b/plugins/dspy-appliance-research-assistant/commands/research-ask.md new file mode 100644 index 0000000..427212a --- /dev/null +++ b/plugins/dspy-appliance-research-assistant/commands/research-ask.md @@ -0,0 +1,11 @@ +--- +description: Run the research-assistant appliance on a question — a ReAct agent (with reflexion) gathers evidence via the tool registry, then a ChainOfThought synthesizer writes a grounded, cited answer with named gaps. Prints the trace and the answer. +argument-hint: "\"\" [--program src/dspy/research-assistant.ts] [--max-steps N] [--no-reflexion] [--show-evidence]" +--- +Run a research query. Parse `$ARGUMENTS` for the question, `--program` (default `src/dspy/research-assistant.ts`), `--max-steps` (override the ReAct budget), `--no-reflexion` (skip lesson recall + episode recording this run), `--show-evidence` (print the gathered notes). + +1. `const ra = await buildResearchAssistant({ maxSteps });` — `ReAct(gather, reflexion) → ChainOfThought(synthesize)`. +2. Before the loop: print the lessons `reflexion.recall(taskKey)` surfaced (and any matched search-strategy skills) — these go into the ReAct thought prompt. +3. `const out = await ra.run({ question });` +4. Print: each ReAct step `{ thought, action:{tool, args}, observation }`; with `--show-evidence`, the gathered notes (`{ id, source, text }`); then the synthesizer's **answer**, **citations** (`{ source, claim }`), and **gaps**. After: what `recordEpisode` did — episode stored, and whether a `react-skill` (a search strategy that's worked ≥ `skillThreshold` times) was promoted. +5. If it loops without gathering useful evidence: the `search`/`fetch` handlers are stubs or your backend isn't returning anything — wire/fix them (the agent can't research what it can't retrieve). If the answer overclaims or cites evidence it didn't gather: that's what `/research-tune` fixes (the metric penalises ungrounded citations). diff --git a/plugins/dspy-appliance-research-assistant/commands/research-init.md b/plugins/dspy-appliance-research-assistant/commands/research-init.md new file mode 100644 index 0000000..4c82225 --- /dev/null +++ b/plugins/dspy-appliance-research-assistant/commands/research-init.md @@ -0,0 +1,11 @@ +--- +description: Scaffold the DSPy.ts research-assistant appliance into your repo — copy the program template to src/dspy/research-assistant.ts (+ spec), create the AgentDB reflexion store, and print next steps to wire your real search/fetch backends and tune. +argument-hint: "[--dest src/dspy/research-assistant.ts] [--reflexion .dspy/research-reflexion] [--max-steps N]" +--- +Set up the research-assistant appliance. Parse `$ARGUMENTS` for `--dest` (default `src/dspy/research-assistant.ts`), `--reflexion` (AgentDB store path, default `.dspy/research-reflexion`), `--max-steps` (ReAct step budget, default 8). + +1. Confirm `dspy.ts` is a dependency. +2. Copy `${CLAUDE_PLUGIN_ROOT}/templates/research-assistant.ts` → `--dest` (+ `research-assistant.spec.ts`); rewrite `REFLEXION_PATH` to `--reflexion`. +3. `const store = new AgentDBClient({ vectorDimension: 384, storage: { path: reflexionPath } }); await store.init();` — create the reflexion store (it starts empty; lessons/skills accrue as you run). +4. Tell the user: the `search` / `fetch` tools in the template are stubs — wire them to your real backend (a web search API, your doc corpus, an `AgentDBClient` retriever, etc.). The `note` tool is the bridge to the synthesizer — the agent must `note(source, text)` every fact it'll cite. +5. Print next steps: implement the tool handlers; `/research-ask ""` to try it; build `tune/research-qa.json` (`[{ input:{question}, output:{mustCover:[...subtopics], answerable} }]` — include an `answerable:false` case); then `/research-tune tune/research-qa.json`. diff --git a/plugins/dspy-appliance-research-assistant/commands/research-tune.md b/plugins/dspy-appliance-research-assistant/commands/research-tune.md new file mode 100644 index 0000000..20e6800 --- /dev/null +++ b/plugins/dspy-appliance-research-assistant/commands/research-tune.md @@ -0,0 +1,11 @@ +--- +description: Tune the research-assistant's synthesizer (and the ReAct thought prompt) with MIPROv2 against a graded research set and the groundedness/coverage metric, with AgentDB experience replay and a CompilationTracer. +argument-hint: " [--program src/dspy/research-assistant.ts] [--target synthesizer|gatherer|both] [--trials N] [--replay .dspy/research-replay] [--cache .dspy/research-cache]" +--- +Optimize the research assistant. Parse `$ARGUMENTS` for the graded set (`[{ input:{question}, output:{mustCover:[...], answerable} }]`), `--program`, `--target` (which module's prompt to tune — default `synthesizer`; `gatherer` tunes the ReAct thought prompt; `both` does two passes), `--trials` (default 12), `--replay` (AgentDB → `replayStore`), `--cache` (AgentDB → `CachingLM`). + +1. Build a *training* program. For `--target synthesizer`: run the real `gatherer` (ReAct) to produce evidence per example, then optimize only the `synthesizer` `ChainOfThought` on `{question, evidence} → {answer, citations, gaps}` via `groundedAnswerMetric`. For `--target gatherer`: optimize the ReAct module's thought-prompt instruction (score the end-to-end answer, so better gathering shows up downstream). `both` = synthesizer pass, then gatherer pass. +2. Optional: `configureLM(new CachingLM(getLM(), { store: cacheClient, similarityThreshold: 0.98, embed: 'model' }))` — research runs are expensive; cache hard. +3. `const store = replayPath ? new AgentDBClient({ vectorDimension: 64, storage: { path: replayPath } }) : undefined; await store?.init(); const tracer = new CompilationTracer({ store });` +4. `const opt = new MIPROv2(groundedAnswerMetric, { numTrials, numCandidateInstructions: 6, replayStore: store, replayTopK: 3, tracer }); const tuned = await opt.compile(trainingProgram, set);` +5. `opt.save('src/dspy/research-assistant.optimized.json');` Report `opt.result` (best score, trials, `warmStarted`), `tracer.causalChain(runId)`, and the delta on a held-out slice. Flat scores ⇒ the graded set / metric — do you have `mustCover` markers and an `answerable:false` case? did the gatherer actually return evidence (a synthesizer can't be grounded if the evidence is empty)? — not the budget. The reflexion store grows on its own across runs; for ongoing prompt evolution, `dspy-evolution`'s `/dspy-evolve`. diff --git a/plugins/dspy-appliance-research-assistant/mcp/server.js b/plugins/dspy-appliance-research-assistant/mcp/server.js new file mode 100644 index 0000000..87c1374 --- /dev/null +++ b/plugins/dspy-appliance-research-assistant/mcp/server.js @@ -0,0 +1,32 @@ +#!/usr/bin/env node +/** + * dspy-appliance-research-assistant MCP server — a pre-wired DSPy.ts research assistant. + * Tools: research_init (scaffold the appliance + create the AgentDB reflexion store), + * research_ask (ReAct gathers evidence via the tool registry + ChainOfThought synthesizes a + * grounded, cited answer), research_tune (MIPROv2-tune the synthesizer / ReAct thought prompt + * against a graded set + the groundedness metric), research_status (reflexion store stats + + * whether a tuned synthesizer is loaded). + * Resources: dspy://research-assistant/template, dspy://research-tool-registry, dspy://synthesis-and-grounding. + * + * Scaffold: handlers shell out to `npx ts-node` against src/dspy/* (the copied appliance) + * and dspy.ts (ReAct, ReActReflexion, ChainOfThought, Pipeline, AgentDBClient, MIPROv2, + * CachingLM, CompilationTracer). Flesh out the @modelcontextprotocol/sdk stdio wiring + handlers. + */ +'use strict'; +const TOOLS = [ + { name: 'research_init', description: 'Scaffold the DSPy.ts research-assistant appliance into a repo: copy the program template to src/dspy/research-assistant.ts (+ spec), create the AgentDB reflexion store. The search/fetch tools are stubs to wire to a real backend; the note tool is the bridge to the synthesizer. Returns created paths.', inputSchema: { type: 'object', properties: { dest: { type: 'string' }, reflexionPath: { type: 'string' }, maxSteps: { type: 'number' } } } }, + { name: 'research_ask', description: 'Run a research query: a ReAct agent (with reflexion) gathers evidence via search/fetch/note, then ChainOfThought synthesizes a grounded, cited answer with named gaps. Returns {answer, citations:[{source,claim}], gaps, steps:[{thought,action,observation}], evidence, recalledLessons, promotedSkill}.', inputSchema: { type: 'object', properties: { question: { type: 'string' }, program: { type: 'string' }, maxSteps: { type: 'number' }, useReflexion: { type: 'boolean' } }, required: ['question'] } }, + { name: 'research_tune', description: 'Tune the research-assistant with MIPROv2 against a graded set ([{input:{question}, output:{mustCover:[...], answerable}}]) and the groundedness/coverage metric. target: synthesizer (cheap) | gatherer (ReAct thought prompt) | both. AgentDB replay + tracer. Saves .optimized.json; returns {bestScore, trials, warmStarted, delta, causalChain}.', inputSchema: { type: 'object', properties: { gradedSet: { type: 'string' }, program: { type: 'string' }, target: { type: 'string', enum: ['synthesizer', 'gatherer', 'both'] }, numTrials: { type: 'number' }, replayPath: { type: 'string' }, cachePath: { type: 'string' } }, required: ['gradedSet'] } }, + { name: 'research_status', description: 'Research-assistant status: reflexion AgentDB store stats (lessons / promoted skills / episodes / tiers / quantization) and whether a tuned synthesizer (.optimized.json) is present.', inputSchema: { type: 'object', properties: { reflexionPath: { type: 'string' }, program: { type: 'string' } } } }, +]; +const RESOURCES = [ + { uri: 'dspy://research-assistant/template', name: 'Research-assistant program template', description: 'The DSPy.ts research-assistant appliance source — ReAct(search/fetch/note, reflexion) → ChainOfThought(synthesize) + groundedAnswerMetric.', mimeType: 'text/typescript' }, + { uri: 'dspy://research-tool-registry', name: 'Research tool registry guide', description: 'Wiring search/fetch/note to a real backend, the note-as-bridge pattern, error handling, step budget.', mimeType: 'text/markdown' }, + { uri: 'dspy://synthesis-and-grounding', name: 'Synthesis & grounding guide', description: 'The groundedness/coverage metric, building a non-overclaiming research set, tuning and validating.', mimeType: 'text/markdown' }, +]; +module.exports = { TOOLS, RESOURCES }; +// TODO: wire @modelcontextprotocol/sdk StdioServerTransport; handlers shell out to +// `npx ts-node` against src/dspy/* and dspy.ts (ReAct, ReActReflexion, ChainOfThought, Pipeline, AgentDBClient, MIPROv2, CachingLM, CompilationTracer). +if (require.main === module) { + process.stderr.write('[dspy-appliance-research-assistant mcp] scaffold — handlers shell out to `npx ts-node` against src/dspy/* and dspy.ts. Tools: ' + TOOLS.map(t => t.name).join(', ') + '\n'); +} diff --git a/plugins/dspy-appliance-research-assistant/skills/research-tool-registry/SKILL.md b/plugins/dspy-appliance-research-assistant/skills/research-tool-registry/SKILL.md new file mode 100644 index 0000000..7bbaf00 --- /dev/null +++ b/plugins/dspy-appliance-research-assistant/skills/research-tool-registry/SKILL.md @@ -0,0 +1,27 @@ +--- +name: research-tool-registry +version: "0.1.0" +author: rUv +tags: [dspy, appliance, research-assistant, react, tools, tool-use] +description: > + How to wire the search / fetch / note tool registry for the DSPy.ts research-assistant appliance — backend choices, the note-as-bridge pattern, error handling, and the ReAct step budget — so the agent gathers good evidence the synthesizer can cite. + Use when: implementing or debugging the tools of a research-assistant (`/research-init`, `/research-ask`). +--- +# The research tool registry + +The appliance's `ReAct` gatherer has three tools. The synthesizer downstream only sees what the gatherer **noted** — so the registry's job is: find evidence → read it → record it with its source. + +## The three tools +- **`search(query)`** — full-text search over your corpus / the web. Returns ~5 snippets with **source ids**. Wire this to: a web search API, your doc corpus, an `AgentDBClient` retriever (`dspy-rag`'s `RetrieveModule`), an internal search API. The description must tell the model what it searches and when *not* to use it (not for arithmetic; not for reading a known source). +- **`fetch(source)`** — full text of a known source id from a prior `search`. Wire to: an HTTP fetch, a doc-store get, `AgentDBClient.searchText`. The agent should `fetch` before citing past a snippet. +- **`note(source, text)`** — the bridge. Records `{ id, source, text }` to a per-run scratchpad; the gatherer's `evidence` output is the concatenation of notes. **Every fact the agent intends to use in the answer must be noted with its source.** Don't change this contract — the synthesizer's groundedness metric checks citations against the noted evidence. + +## Mechanics +- **Handlers must not throw.** An exception becomes the observation; the agent then reasons about a stack trace. Catch; return a short, actionable string ("error: `query` is required", "fetch failed: doc:42 not found"). +- **Observations terse and parseable.** The whole observation goes back into context every step. Return the snippet/answer, not a giant JSON payload; truncate/summarise. +- **Stable, semantic source ids** — `doc:42`, `https://...`, `kb/security#reset`. The citation only works if the id the agent saw in `search` is the id it `note`s and the id the synthesizer can verify. +- **Idempotent search/fetch** — ReAct may retry. `note` is append-only and that's fine. +- **Step budget** — `maxSteps` 6–10. Research is iterative (search → fetch → note, a few times). If it routinely hits the cap without enough notes, the tools are returning junk or the thought prompt doesn't explain the task — read a `/research-ask` trace; don't just raise the cap. + +## Reflexion interaction +`ReActReflexion` recalls *lessons* (e.g. "for questions about X, search for Y first") and promotes *search strategies* (recurring successful tool sequences) into skills — injected into the thought prompt before the loop. Use a `taskKey` scoped to the research-task *type*, not the individual question, so lessons generalise. (See `dspy-react`'s `reflexion-loop` skill.) diff --git a/plugins/dspy-appliance-research-assistant/skills/synthesis-and-grounding/SKILL.md b/plugins/dspy-appliance-research-assistant/skills/synthesis-and-grounding/SKILL.md new file mode 100644 index 0000000..83cfa29 --- /dev/null +++ b/plugins/dspy-appliance-research-assistant/skills/synthesis-and-grounding/SKILL.md @@ -0,0 +1,32 @@ +--- +name: synthesis-and-grounding +version: "0.1.0" +author: rUv +tags: [dspy, appliance, research-assistant, synthesis, grounding, citations, metric] +description: > + How the DSPy.ts research-assistant synthesizes a grounded, cited answer from gathered evidence, how the groundedness/coverage metric scores it, and how to build a research set that doesn't teach it to overclaim. + Use when: tuning the synthesizer (`/research-tune`), building its graded set, or validating answers. +--- +# Synthesis & grounding + +The gatherer hands the synthesizer a bundle of notes (`{ source, text }`). The synthesizer (`ChainOfThought`) writes the answer, the `citations` (every claim → a source id), and `gaps` (sub-questions the evidence didn't answer). + +## What "good" means here +- **Grounded** — every substantive claim is backed by a citation, and every cited `source` actually appears in the gathered evidence. A citation to evidence that wasn't gathered scores *worse* than no citation (it's a hallucinated source). Uncited claims are capped (~0.3 on the groundedness axis). +- **Covers the question** — touches the required sub-topics (`gold.mustCover`). A polished answer to half the question isn't good. +- **Calibrated** — on thin/conflicting evidence, says so and lists `gaps`; doesn't paper over. When the evidence genuinely can't answer (`gold.answerable === false`), an honest "the evidence is insufficient" is the *correct* output (scores ~1); a confident fabricated answer scores ~0.1. + +`groundedAnswerMetric` ≈ `0.4 × grounded + 0.4 × coverage + 0.2 × calibration`. + +## Building the graded set (the part people get wrong) +- Questions with `mustCover` sub-topic markers (the metric checks the answer mentions them) and a `verdict`/`answerable` flag. +- **Include `answerable:false` cases** — questions your sources can't answer. ~15–25%. Without them every example rewards producing an answer, and tuning yields an assistant that *always* answers — i.e. fabricates when it shouldn't. This is the single most important thing in the set. +- Span the kinds of questions you care about (factual lookup, multi-hop synthesis, "compare X and Y"); the tuned assistant is only as good as what's represented. +- Hold out a slice; never tune on it. +- Goodhart watch: the metric can be gamed (citation-stuff, keyword-drop the `mustCover` terms). Spot-check top-scoring answers — do they read like a careful researcher's, or like metric-bait? + +## Tuning & validating +- `/research-tune ` — MIPROv2 against `groundedAnswerMetric`, `replayStore` + a `CachingLM`. Tune `--target synthesizer` first (cheap: run the real gatherer once per example, then optimize the synthesizer prompt); tune `--target gatherer` (the ReAct thought prompt) if gathering is the weak link. Read `tracer.causalChain(runId)`. +- Flat scores ⇒ the graded set / metric, or the gatherer is returning empty evidence (a synthesizer can't be grounded on nothing — fix the tools first). Not the budget. +- Ongoing evolution across iterations: `dspy-evolution`'s `/dspy-evolve` with a held-out research benchmark. +- Validate with `/research-ask` on held-out questions: claims backed by gathered evidence (real `source` ids)? sub-topics covered? gaps named? hedges when evidence is thin? diff --git a/plugins/dspy-appliance-research-assistant/templates/research-assistant.spec.ts b/plugins/dspy-appliance-research-assistant/templates/research-assistant.spec.ts new file mode 100644 index 0000000..fd78aa8 --- /dev/null +++ b/plugins/dspy-appliance-research-assistant/templates/research-assistant.spec.ts @@ -0,0 +1,22 @@ +import { configureLM, DummyLM } from 'dspy.ts'; +import { synthesizer, groundedAnswerMetric } from './research-assistant'; + +describe('research-assistant appliance', () => { + beforeEach(async () => { const lm = new DummyLM(); await lm.init(); configureLM(lm); }); + + it('synthesizer runs and returns an answer', async () => { + const out = await synthesizer.run({ question: 'What is X?', evidence: 'n1 (source: doc:1): X is a thing.' }); + expect(typeof out.answer).toBe('string'); + }); + + it('groundedAnswerMetric rewards citations that reference gathered evidence', () => { + const grounded = groundedAnswerMetric({ question: 'q', evidence: 'n1 (source: doc:1): fact' }, { answer: 'fact', citations: [{ source: 'doc:1', claim: 'fact' }] }); + const hallucinated = groundedAnswerMetric({ question: 'q', evidence: 'n1 (source: doc:1): fact' }, { answer: 'fact', citations: [{ source: 'doc:999', claim: 'fact' }] }); + expect(grounded).toBeGreaterThan(hallucinated); + }); + + it('groundedAnswerMetric rewards hedging when there is no evidence', () => { + const honest = groundedAnswerMetric({ question: 'q', evidence: '' }, { answer: "The evidence is insufficient.", citations: [], gaps: ['everything'] }, { answerable: false }); + expect(honest).toBeGreaterThan(0.9); + }); +}); diff --git a/plugins/dspy-appliance-research-assistant/templates/research-assistant.ts b/plugins/dspy-appliance-research-assistant/templates/research-assistant.ts new file mode 100644 index 0000000..a9ece28 --- /dev/null +++ b/plugins/dspy-appliance-research-assistant/templates/research-assistant.ts @@ -0,0 +1,132 @@ +/** + * DSPy.ts research-assistant appliance — a ReAct agent (search / fetch / note tools) with + * ReActReflexion, feeding a ChainOfThought synthesizer that writes a grounded, cited answer. + * Copy this into your repo (e.g. src/dspy/research-assistant.ts), plug your real search/fetch + * into the tool handlers, point REFLEXION_PATH at an AgentDB store, configure a real LM, then + * tune `synthesizer` (and the ReAct thought prompt) with MIPROv2 against `groundedAnswerMetric`. + * + * Requires: dspy.ts@^2.2.0 + */ +import { + ReAct, ReActReflexion, ChainOfThought, Pipeline, AgentDBClient, + type Signature, type MetricFunction, type TrainingExample, +} from 'dspy.ts'; + +export const REFLEXION_PATH = process.env.RESEARCH_REFLEXION_PATH ?? '.dspy/research-reflexion'; + +/** Tool registry. Replace the handler bodies with your real search / fetch / store. */ +export function researchTools() { + // a per-run scratchpad of gathered evidence (id -> { source, text }) + const notes: { id: string; source: string; text: string }[] = []; + return [ + { + name: 'search', + description: 'search(query: string) — full-text search over your corpus / the web. Returns up to 5 result snippets with their source ids. Use to find evidence for a sub-question. Not for arithmetic; not for reading a known source (use fetch).', + handler: async (args: { query: string }) => { + if (!args?.query) return 'error: `query` is required'; + // TODO: real search. Stub returns nothing useful. + return `search("${args.query}") → (no results — wire a real search backend). Source ids would look like "doc:42".`; + }, + }, + { + name: 'fetch', + description: 'fetch(source: string) — retrieve the full text of a known source id (e.g. "doc:42") returned by search. Use to read past the snippet before you cite it.', + handler: async (args: { source: string }) => { + if (!args?.source) return 'error: `source` is required'; + // TODO: real fetch. + return `fetch("${args.source}") → (no content — wire a real fetch backend).`; + }, + }, + { + name: 'note', + description: 'note(source: string, text: string) — record a piece of evidence (a quote/fact + the source id it came from) to the scratchpad so the synthesizer can use and cite it. Call this for every fact you intend to use in the answer.', + handler: async (args: { source: string; text: string }) => { + if (!args?.source || !args?.text) return 'error: both `source` and `text` are required'; + const id = `n${notes.length + 1}`; + notes.push({ id, source: args.source, text: args.text }); + return `noted ${id} (source: ${args.source})`; + }, + }, + ]; +} + +export const synthesizerSignature: Signature = { + inputs: [ + { name: 'question', type: 'string', required: true }, + { name: 'evidence', type: 'string', required: true, description: 'the gathered notes: pieces of evidence each tagged with its source id' }, + ], + outputs: [ + { name: 'answer', type: 'string', required: true, description: "a grounded answer; if the evidence is thin or conflicting, say so — don't overclaim" }, + { name: 'citations', type: 'object', required: true, description: 'array of { source, claim } — every substantive claim mapped to the source id that supports it' }, + { name: 'gaps', type: 'object', required: false, description: 'array of sub-questions the evidence did not answer' }, + ], +}; + +const synthesizerPrompt = (i: { question: string; evidence: string }) => [ + 'You are a research assistant. Write the answer to the question using ONLY the evidence below.', + 'Every substantive claim must be backed by a citation to the source id of the evidence that supports it. Do not cite evidence you did not use; do not state things the evidence does not support.', + "If the evidence is thin, conflicting, or doesn't cover part of the question, say so and list the gaps — don't paper over them.", + '', + `Evidence:\n${i.evidence}`, + '', + `Question: ${i.question}`, + 'Answer:', +].join('\n'); + +/** ChainOfThought synthesizer — the module you tune. */ +export const synthesizer = new ChainOfThought({ + name: 'ResearchSynthesizer', + signature: synthesizerSignature, + promptTemplate: synthesizerPrompt, +}); + +/** The full appliance: ReAct(gather evidence, with reflexion) → ChainOfThought(synthesize). `run({ question })` → { answer, citations, gaps, steps, recalledLessons }. */ +export async function buildResearchAssistant(opts?: { maxSteps?: number }): Promise { + const store = new AgentDBClient({ vectorDimension: 384, storage: { path: REFLEXION_PATH } }); + await store.init(); + const reflexion = new ReActReflexion({ store, recallK: 3, skillThreshold: 3 }); + const gatherSignature: Signature = { + inputs: [{ name: 'question', type: 'string', required: true }], + outputs: [{ name: 'evidence', type: 'string', required: true, description: 'the notes recorded via the note tool, concatenated' }], + }; + const gatherer = new ReAct({ + name: 'ResearchGatherer', + signature: gatherSignature, + tools: researchTools(), + maxSteps: opts?.maxSteps ?? 8, + reflexion, // taskKey defaults from the signature/name — keep it stable per research-task type + }); + return new Pipeline([ + { module: gatherer }, + { module: synthesizer }, + ] as any); +} + +/** Metric for tuning: grounded (citations point at used evidence), covers the question, calibrated about gaps — not a confident hallucination, not a non-answer. */ +export const groundedAnswerMetric: MetricFunction = ( + input: { question: string; evidence: string }, + out: { answer?: string; citations?: { source: string; claim?: string }[]; gaps?: unknown[] }, + gold?: { mustCover?: string[]; answerable?: boolean }, +) => { + if (!out?.answer) return 0; + const said = out.answer.toLowerCase(); + const hedged = /\b(thin|conflicting|insufficient|couldn'?t find|no evidence|unclear)\b/.test(said) || (Array.isArray(out.gaps) && out.gaps.length > 0); + if (gold?.answerable === false) return hedged ? 1 : 0.1; // honest about no evidence + // groundedness: citations exist and reference evidence that's actually in the gathered notes + const cited = out.citations ?? []; + const grounded = cited.length === 0 ? 0.3 + : cited.every((c) => input.evidence.includes(c.source)) ? 1.0 + : 0.15; // cited something not gathered + // coverage: did the answer touch the sub-questions we required? + let coverage = 0.6; + if (gold?.mustCover?.length) { + const hit = gold.mustCover.filter((t) => said.includes(t.toLowerCase())).length; + coverage = hit / gold.mustCover.length; + } + // calibration: a flat "here's the answer" with zero gaps when the question is broad is suspicious; mild bonus for naming gaps + const calibrated = Array.isArray(out.gaps) ? Math.min(1, 0.7 + 0.3 * Math.sign(out.gaps.length)) : 0.7; + return 0.4 * grounded + 0.4 * coverage + 0.2 * calibrated; +}; + +/** Tuning example shape. */ +export type ResearchExample = TrainingExample<{ question: string }, { mustCover?: string[]; answerable?: boolean }>; diff --git a/plugins/dspy-appliance-support-bot/.claude-plugin/plugin.json b/plugins/dspy-appliance-support-bot/.claude-plugin/plugin.json new file mode 100644 index 0000000..a49d99d --- /dev/null +++ b/plugins/dspy-appliance-support-bot/.claude-plugin/plugin.json @@ -0,0 +1,10 @@ +{ + "name": "dspy-appliance-support-bot", + "description": "Vertical appliance: a pre-wired DSPy.ts support assistant. Ships a ready-to-run program — RetrieveModule (MMR) over an AgentDB knowledge base → ChainOfThought answer with citations → a quality metric (helpfulness + groundedness + honest 'I don't know') → MIPROv2 tuning. Bring your own KB; commands scaffold it into your repo, answer questions, and tune the answerer.", + "version": "0.1.0", + "author": { "name": "rUv", "url": "https://github.com/ruvnet" }, + "homepage": "https://github.com/ruvnet/dspy.ts", + "license": "MIT", + "keywords": ["dspy", "dspy.ts", "appliance", "support-bot", "rag", "knowledge-base", "agentdb", "miprov2"], + "mcpServers": { "dspy-appliance-support-bot": { "command": "node", "args": ["${CLAUDE_PLUGIN_ROOT}/mcp/server.js"] } } +} diff --git a/plugins/dspy-appliance-support-bot/AGENTS.md b/plugins/dspy-appliance-support-bot/AGENTS.md new file mode 100644 index 0000000..3746e31 --- /dev/null +++ b/plugins/dspy-appliance-support-bot/AGENTS.md @@ -0,0 +1,13 @@ +# dspy-appliance-support-bot — for Codex / MCP clients + +A vertical appliance: a pre-wired DSPy.ts support assistant. Ships a ready-to-run program (`templates/support-bot.ts` — `RetrieveModule` MMR over an AgentDB KB → `ChainOfThought` answer with citations + a `supportMetric`) plus tooling to stand it up. MCP server: `mcp/server.js`, command `node ${CLAUDE_PLUGIN_ROOT}/mcp/server.js`. + +**Tools** +- `support_bot_init` — copy the appliance program into `src/dspy/support-bot.ts` (+ spec), create the AgentDB knowledge base, optionally index a docs dir/glob. +- `support_bot_ask` — ask a question: retrieves KB passages (MMR), answers with ChainOfThought grounded in them. Returns `{answer, citations, passages, context}`. +- `support_bot_tune` — MIPROv2-tune the answerer against a Q/A set (`[{input:{question}, output:{answer?, answerable?}}]` — include `answerable:false` cases) and the quality metric (helpfulness + groundedness + honest "I don't know"); AgentDB replay + tracer; saves `.optimized.json`. +- `support_bot_status` — KB stats + whether a tuned answerer is loaded. + +**Resources**: `dspy://support-bot/template`, `dspy://support-kb-curation`, `dspy://support-answer-quality`. + +Handlers shell out to `npx ts-node` against `src/dspy/*` (the copied appliance) and `dspy.ts`. Slash commands `/support-bot-init`, `/support-bot-ask`, `/support-bot-tune` wrap the same flows; the `support-bot-builder` agent stands up and tunes a bot for a specific KB. Builds on `dspy-rag` (index/retrieve), `dspy-optimize` (MIPROv2), `dspy-observability` (tracer, cache), and `dspy-evolution` (`/dspy-evolve` for ongoing prompt evolution). diff --git a/plugins/dspy-appliance-support-bot/agents/support-bot-builder.md b/plugins/dspy-appliance-support-bot/agents/support-bot-builder.md new file mode 100644 index 0000000..2e7548b --- /dev/null +++ b/plugins/dspy-appliance-support-bot/agents/support-bot-builder.md @@ -0,0 +1,14 @@ +--- +name: support-bot-builder +description: Stands up and tunes the DSPy.ts support-bot appliance for a specific product/KB — scaffolds the program, curates and indexes the knowledge base, builds the Q/A tuning set (including honesty cases), runs MIPROv2/GEPA against the quality metric, and validates answer quality. Use to go from "we have these docs, build us a support assistant" to a tuned, grounded bot. +--- +You build support-bot appliances on DSPy.ts. + +STEPS: +1. **Scaffold** — `/support-bot-init --kb --docs `: copies the appliance program (`RetrieveModule → ChainOfThought` + `supportMetric`) into `src/dspy/support-bot.ts`, creates the AgentDB KB. The appliance code is the starting point — adjust the answerer prompt to the product's voice if needed, but keep the "answer only from context / cite / say I don't know" constraints. +2. **Curate the KB** — chunk on natural boundaries, keep `{ source, title, section }` metadata so the bot can cite, dedup, and prefer `tier: 'long'`. Index with `dspy-rag`'s `/dspy-index`. (See the `support-kb-curation` skill.) The bot can only answer what's in the KB; gaps become "I don't know" — that's correct behavior, fill them by indexing more. +3. **Q/A tuning set** — collect real questions with gold answers; **include `answerable:false` cases** (questions the KB deliberately doesn't cover) so tuning rewards honest punts, not bluffing. Aim for coverage of the question space, not just FAQs. +4. **Tune** — `/support-bot-tune `: MIPROv2 against `supportMetric` (helpfulness × groundedness, "I don't know" = correct when unanswerable), with `replayStore` + a `CachingLM`. Read the trace. If you want the prompt to keep improving over time, hand off to `dspy-evolution`'s `/dspy-evolve` with a held-out benchmark. +5. **Validate** — `/support-bot-ask` on a held-out set: are answers grounded? are citations real (in the retrieved context)? does it punt when it should? Use `/dspy-retrieve` to debug retrieval misses. (See the `support-answer-quality` skill.) + +DELIVER: the program file, the indexed KB, the Q/A set, the tuned answerer (`.optimized.json`), and a quality report (mean metric, examples of grounded vs punted answers, any known KB gaps). diff --git a/plugins/dspy-appliance-support-bot/commands/support-bot-ask.md b/plugins/dspy-appliance-support-bot/commands/support-bot-ask.md new file mode 100644 index 0000000..caaabc3 --- /dev/null +++ b/plugins/dspy-appliance-support-bot/commands/support-bot-ask.md @@ -0,0 +1,10 @@ +--- +description: Ask the support-bot appliance a question — retrieves KB passages (MMR), answers with ChainOfThought grounded in them, and shows the answer, citations, and the passages used. +argument-hint: "\"\" [--program src/dspy/support-bot.ts] [--k N] [--show-context]" +--- +Run the support bot. Parse `$ARGUMENTS` for the question, `--program` (default `src/dspy/support-bot.ts`), `--k` (override passages), `--show-context` (print the assembled context). + +1. `const bot = await buildSupportBot({ k });` (from the appliance module) — a `Pipeline` of `RetrieveModule → ChainOfThought`. +2. `const out = await bot.run({ question });` +3. Print: the **answer**; the **citations** (`out.citations`); the **passages** used (rank · score · source · snippet); with `--show-context`, the assembled context string the answerer saw. +4. If the answer is "I don't know" but you expected coverage: the KB is missing it or retrieval isn't surfacing it — `/dspy-retrieve ""` (from `dspy-rag`) to debug, then index more docs. If the answer is confident but wrong/uncited: that's what `/support-bot-tune` fixes (the metric penalises uncited/hallucinated answers). diff --git a/plugins/dspy-appliance-support-bot/commands/support-bot-init.md b/plugins/dspy-appliance-support-bot/commands/support-bot-init.md new file mode 100644 index 0000000..d871618 --- /dev/null +++ b/plugins/dspy-appliance-support-bot/commands/support-bot-init.md @@ -0,0 +1,11 @@ +--- +description: Scaffold the DSPy.ts support-bot appliance into your repo — copy the program template to src/dspy/support-bot.ts (+ spec), create the AgentDB knowledge base, and print the next steps to index docs and tune. +argument-hint: "[--dest src/dspy/support-bot.ts] [--kb .dspy/support-kb] [--docs path/to/docs]" +--- +Set up the support-bot appliance. Parse `$ARGUMENTS` for `--dest` (default `src/dspy/support-bot.ts`), `--kb` (AgentDB corpus path, default `.dspy/support-kb`), `--docs` (a directory/glob of knowledge-base documents to index now, optional). + +1. Confirm `dspy.ts` is a dependency (`npm i dspy.ts` if not). +2. Copy `${CLAUDE_PLUGIN_ROOT}/templates/support-bot.ts` → `--dest`, and `support-bot.spec.ts` next to it. Rewrite `KB_PATH` / the env default to `--kb`. +3. `const kb = new AgentDBClient({ vectorDimension: 384, storage: { path: kbPath } }); await kb.init();` — create the corpus. +4. If `--docs`: chunk + embed + store them into the KB (the same flow as `dspy-rag`'s `/dspy-index --tier long`). Otherwise tell the user to run `/dspy-index ` (from the `dspy-rag` plugin) when ready. +5. Print next steps: `/support-bot-ask ""` to try it; create a `tune/support-qa.json` (`[{ input:{question}, output:{answer?, answerable?} }]` — include some `answerable:false` cases to test honesty); then `/support-bot-tune tune/support-qa.json`. diff --git a/plugins/dspy-appliance-support-bot/commands/support-bot-tune.md b/plugins/dspy-appliance-support-bot/commands/support-bot-tune.md new file mode 100644 index 0000000..04ef911 --- /dev/null +++ b/plugins/dspy-appliance-support-bot/commands/support-bot-tune.md @@ -0,0 +1,12 @@ +--- +description: Tune the support-bot's answerer with MIPROv2 against a Q/A set and the appliance's quality metric (helpfulness + groundedness + honest "I don't know"), with AgentDB experience replay and a CompilationTracer. +argument-hint: " [--program src/dspy/support-bot.ts] [--trials N] [--replay .dspy/support-replay] [--cache .dspy/support-cache]" +--- +Optimize the `answerer` `ChainOfThought` in `src/dspy/support-bot.ts`. Parse `$ARGUMENTS` for the Q/A set (`[{ input:{question}, output:{answer?, answerable?} }]` — include `answerable:false` cases), `--program`, `--trials` (default 12), `--replay` (AgentDB path → `replayStore`, warm-starts re-tunes), `--cache` (AgentDB path → wrap the LM in `CachingLM`). + +1. Build a *training* program: `RetrieveModule(KB) → answerer`, so each Q/A example is scored end-to-end (retrieval + answer) via `supportMetric` from the appliance module. +2. Optional: `configureLM(new CachingLM(getLM(), { store: cacheClient, similarityThreshold: 0.98, embed: 'model' }))`. +3. `const store = replayPath ? new AgentDBClient({ vectorDimension: 64, storage: { path: replayPath } }) : undefined; await store?.init(); const tracer = new CompilationTracer({ store });` +4. `const opt = new MIPROv2(supportMetric, { numTrials, numCandidateInstructions: 6, replayStore: store, replayTopK: 3, tracer }); const tuned = await opt.compile(trainingProgram, qaSet);` +5. `opt.save('src/dspy/support-bot.optimized.json');` — load it in `support-bot-ask` via `opt.load(...)` to swap in the tuned answerer. +6. Report `opt.result` (best score, trials, `warmStarted`), the trial trace (`tracer.causalChain(runId)`), and the score delta vs the raw answerer on a held-out slice. If scores are flat, look at the metric/Q-A set before adding trials (and make sure you've got `answerable:false` cases — without them the bot learns to always answer). For more aggressive prompt evolution, run `dspy-evolution`'s `/dspy-evolve` against a held-out benchmark instead. diff --git a/plugins/dspy-appliance-support-bot/mcp/server.js b/plugins/dspy-appliance-support-bot/mcp/server.js new file mode 100644 index 0000000..4f50b6c --- /dev/null +++ b/plugins/dspy-appliance-support-bot/mcp/server.js @@ -0,0 +1,32 @@ +#!/usr/bin/env node +/** + * dspy-appliance-support-bot MCP server — a pre-wired DSPy.ts support assistant. + * Tools: support_bot_init (scaffold the appliance into a repo + create the AgentDB KB), + * support_bot_ask (retrieve KB passages + ChainOfThought answer with citations), + * support_bot_tune (MIPROv2-tune the answerer against a Q/A set + the quality metric), + * support_bot_status (KB stats + whether a tuned answerer is loaded). + * Resources: dspy://support-bot/template (the program template), dspy://support-kb-curation, + * dspy://support-answer-quality. + * + * Scaffold: handlers shell out to `npx ts-node` against src/dspy/* (the copied appliance) + * and dspy.ts (RetrieveModule, ChainOfThought, Pipeline, AgentDBClient, MIPROv2, + * CachingLM, CompilationTracer). Flesh out the @modelcontextprotocol/sdk stdio wiring + handlers. + */ +'use strict'; +const TOOLS = [ + { name: 'support_bot_init', description: 'Scaffold the DSPy.ts support-bot appliance into a repo: copy the program template to src/dspy/support-bot.ts (+ spec), create the AgentDB knowledge base, optionally index a docs dir/glob. Returns the created paths + KB stats.', inputSchema: { type: 'object', properties: { dest: { type: 'string' }, kbPath: { type: 'string' }, docs: { type: 'string', description: 'dir | glob of KB documents to index now' } } } }, + { name: 'support_bot_ask', description: 'Ask the support-bot a question: retrieves KB passages (MMR), answers with ChainOfThought grounded in them. Returns {answer, citations, passages:[{rank,score,source,snippet}], context}.', inputSchema: { type: 'object', properties: { question: { type: 'string' }, program: { type: 'string' }, k: { type: 'number' } }, required: ['question'] } }, + { name: 'support_bot_tune', description: 'Tune the support-bot answerer with MIPROv2 against a Q/A set ([{input:{question}, output:{answer?, answerable?}}]) and the appliance quality metric (helpfulness + groundedness + honest "I don\'t know"); AgentDB replay store + CompilationTracer. Saves .optimized.json; returns {bestScore, trials, warmStarted, delta, causalChain}.', inputSchema: { type: 'object', properties: { qaSet: { type: 'string' }, program: { type: 'string' }, numTrials: { type: 'number' }, replayPath: { type: 'string' }, cachePath: { type: 'string' } }, required: ['qaSet'] } }, + { name: 'support_bot_status', description: 'Support-bot status: KB AgentDB stats (vectors, dimension, tier counts, quantization), and whether a tuned answerer (.optimized.json) is present.', inputSchema: { type: 'object', properties: { kbPath: { type: 'string' }, program: { type: 'string' } } } }, +]; +const RESOURCES = [ + { uri: 'dspy://support-bot/template', name: 'Support-bot program template', description: 'The DSPy.ts support-bot appliance source — RetrieveModule → ChainOfThought + supportMetric.', mimeType: 'text/typescript' }, + { uri: 'dspy://support-kb-curation', name: 'Support KB curation guide', description: 'Chunking, citation metadata, coverage, tiers for a support-bot knowledge base.', mimeType: 'text/markdown' }, + { uri: 'dspy://support-answer-quality', name: 'Support answer quality guide', description: 'The quality metric, building a non-bluffing Q/A set, tuning and validating.', mimeType: 'text/markdown' }, +]; +module.exports = { TOOLS, RESOURCES }; +// TODO: wire @modelcontextprotocol/sdk StdioServerTransport; handlers shell out to +// `npx ts-node` against src/dspy/* and dspy.ts (RetrieveModule, ChainOfThought, Pipeline, AgentDBClient, MIPROv2, CachingLM, CompilationTracer). +if (require.main === module) { + process.stderr.write('[dspy-appliance-support-bot mcp] scaffold — handlers shell out to `npx ts-node` against src/dspy/* and dspy.ts. Tools: ' + TOOLS.map(t => t.name).join(', ') + '\n'); +} diff --git a/plugins/dspy-appliance-support-bot/skills/support-answer-quality/SKILL.md b/plugins/dspy-appliance-support-bot/skills/support-answer-quality/SKILL.md new file mode 100644 index 0000000..72d3b34 --- /dev/null +++ b/plugins/dspy-appliance-support-bot/skills/support-answer-quality/SKILL.md @@ -0,0 +1,29 @@ +--- +name: support-answer-quality +version: "0.1.0" +author: rUv +tags: [dspy, appliance, support-bot, metric, groundedness, honesty] +description: > + How to judge and tune support-bot answer quality with the appliance's metric — helpfulness, groundedness/citations, and honest "I don't know" — and how to build a Q/A set that doesn't teach the bot to bluff. + Use when: tuning the support bot (`/support-bot-tune`), building its Q/A set, or validating answers. +--- +# Support-bot answer quality + +The appliance ships `supportMetric` — roughly `0.6 × helpfulness + 0.4 × groundedness`, with a special case: when the KB *can't* answer (`gold.answerable === false`), an honest "I don't know" scores ~1 and a confident answer scores ~0.1. + +## What "good" means here +- **Helpful** — actually answers the question. Graded (exact → contains → partial), not binary, so the optimizer has a gradient. (See `dspy-core`'s `metric-design`.) +- **Grounded** — claims trace to the retrieved context; the `citations` output names sources that appear in `context`. A hallucinated citation scores *worse* than no citation. An uncited-but-correct answer is capped (~0.4 on the groundedness axis) — it can't max out. +- **Honest** — "I don't know, try X" is the *correct* output when the KB lacks the answer. The metric and the prompt both allow it; don't penalise it, and don't train it away. + +## Building the Q/A tuning set (the part people get wrong) +- Real questions with gold answers — span the question space, not just the top-10 FAQs. +- **Include `answerable:false` cases** — questions the KB deliberately doesn't cover. Without these, every example rewards answering, and tuning will teach the bot to *always* answer (i.e. bluff on the ones it shouldn't). 15–30% unanswerable is reasonable. +- Hold out a slice for validation; never tune on it. +- Watch Goodhart: if the metric over-rewards citations, the bot will citation-stuff. Spot-check top-scoring answers by hand. + +## Tuning & validating +- `/support-bot-tune ` — MIPROv2 against `supportMetric`, with `replayStore` (warm-start re-tunes) and a `CachingLM` (the search makes many near-identical prompts). Read `tracer.causalChain(runId)` — which instruction change moved the score? +- Flat scores ⇒ metric/Q-A set, not budget. Check: graded metric? `answerable:false` cases present? leakage between tune/validate? +- Want the prompt to keep improving across releases? `dspy-evolution`'s `/dspy-evolve` against a held-out benchmark. +- Validate with `/support-bot-ask` on the held-out slice: grounded? real citations? punts when it should? Debug retrieval misses with `dspy-rag`'s `/dspy-retrieve`. diff --git a/plugins/dspy-appliance-support-bot/skills/support-kb-curation/SKILL.md b/plugins/dspy-appliance-support-bot/skills/support-kb-curation/SKILL.md new file mode 100644 index 0000000..c3277ff --- /dev/null +++ b/plugins/dspy-appliance-support-bot/skills/support-kb-curation/SKILL.md @@ -0,0 +1,26 @@ +--- +name: support-kb-curation +version: "0.1.0" +author: rUv +tags: [dspy, appliance, support-bot, rag, knowledge-base, agentdb] +description: > + How to curate the AgentDB knowledge base behind the DSPy.ts support-bot appliance — chunking, metadata for citations, coverage, deduplication, and tiers — so the bot answers accurately and admits gaps. + Use when: building/extending a support-bot KB (`/support-bot-init`, `/dspy-index`). +--- +# Curating a support-bot KB + +The bot retrieves chunks from this AgentDB corpus and answers *only* from them. So the KB defines what it can answer — and what it correctly says "I don't know" to. + +## Rules of thumb +- **Chunk on natural units** — a help-center article section, a FAQ entry, a runbook step group — ~500–1000 chars, ~10–20% overlap. Don't slice mid-procedure. (See `dspy-rag`'s `chunking-strategy`.) +- **Metadata is for citations.** Store `{ source, title, section, url? }` with every chunk. The appliance's answerer outputs `citations` and the `supportMetric` checks them against the retrieved context — no metadata, no real citations. +- **Cover the question space, not the doc tree.** Audit against real support tickets: every common question should map to ≥1 chunk. Uncovered questions become "I don't know" — acceptable, but track the list and fill it. +- **Dedup.** Near-identical chunks (the same answer in three articles) waste the `k` budget; MMR mitigates at query time but it's cheaper not to store them. +- **Tiers.** Index into `tier: 'long'`. `promote(...)` chunks that get hit constantly into `working`. `evictTier('short', { maxAgeMs })` for transient notices (incidents, temporary workarounds) so stale ops don't leak into answers. +- **Big KB ⇒ `quantization: 'rabitq'`** (`coarseThenRerank`, `rerankFactor ≥ 3`) — ~32× smaller, still fast. +- **Versioned / time-sensitive content** — put a date in the metadata; consider re-indexing on doc updates rather than appending, so old answers don't compete with new ones. + +## Smell tests +- Bot punts on a question you know is documented ⇒ chunk missing, mis-chunked, or not surfacing — `/dspy-retrieve ""` to see the top-k. +- Bot answers confidently but the cited source doesn't actually say that ⇒ chunk boundaries split the qualifier off the claim; re-chunk with more overlap. +- Bot answers from a stale chunk ⇒ evict/re-index that content; add a date to metadata. diff --git a/plugins/dspy-appliance-support-bot/templates/support-bot.spec.ts b/plugins/dspy-appliance-support-bot/templates/support-bot.spec.ts new file mode 100644 index 0000000..277a66b --- /dev/null +++ b/plugins/dspy-appliance-support-bot/templates/support-bot.spec.ts @@ -0,0 +1,22 @@ +import { configureLM, DummyLM } from 'dspy.ts'; +import { answerer, supportMetric } from './support-bot'; + +describe('support-bot appliance', () => { + beforeEach(async () => { const lm = new DummyLM(); await lm.init(); configureLM(lm); }); + + it('answerer runs and returns an answer string', async () => { + const out = await answerer.run({ question: 'How do I reset my password?', context: 'Go to Settings > Security > Reset password. (source: kb/security)' }); + expect(typeof out.answer).toBe('string'); + }); + + it('supportMetric rewards an honest "I don\'t know" when the KB cannot answer', () => { + const s = supportMetric({ question: 'q', context: 'unrelated' }, { answer: "I don't know — that isn't in our knowledge base." }, { answerable: false }); + expect(s).toBeGreaterThan(0.9); + }); + + it('supportMetric penalises a hallucinated citation', () => { + const grounded = supportMetric({ question: 'q', context: 'A: see kb/x' }, { answer: 'A', citations: [{ source: 'kb/x' }] }); + const hallucinated = supportMetric({ question: 'q', context: 'A: see kb/x' }, { answer: 'A', citations: [{ source: 'kb/nope' }] }); + expect(grounded).toBeGreaterThan(hallucinated); + }); +}); diff --git a/plugins/dspy-appliance-support-bot/templates/support-bot.ts b/plugins/dspy-appliance-support-bot/templates/support-bot.ts new file mode 100644 index 0000000..f9b8b91 --- /dev/null +++ b/plugins/dspy-appliance-support-bot/templates/support-bot.ts @@ -0,0 +1,103 @@ +/** + * DSPy.ts support-bot appliance — RetrieveModule (MMR over an AgentDB KB) → ChainOfThought + * answer with citations. Copy this into your repo (e.g. src/dspy/support-bot.ts), point + * KB_PATH at an AgentDB corpus (build it with `dspy-rag`'s /dspy-index), configure a real + * LM, then tune `answerer` with MIPROv2 against the `supportMetric` below. + * + * Requires: dspy.ts@^2.2.0 + */ +import { + RetrieveModule, ChainOfThought, Pipeline, AgentDBClient, + type Signature, type MetricFunction, type TrainingExample, +} from 'dspy.ts'; + +export const KB_PATH = process.env.SUPPORT_KB_PATH ?? '.dspy/support-kb'; + +/** A knowledge-base corpus. Build/extend it with `/dspy-index `. */ +export async function openKB(): Promise { + const kb = new AgentDBClient({ + vectorDimension: 384, // ONNX embeddings; falls back to hashEmbed + storage: { path: KB_PATH }, + performance: { batchEnabled: true /*, quantization: 'rabitq', rerankFactor: 3 */ }, + }); + await kb.init(); + return kb; +} + +export const answererSignature: Signature = { + inputs: [ + { name: 'question', type: 'string', required: true }, + { name: 'context', type: 'string', required: true, description: 'retrieved KB passages' }, + ], + outputs: [ + { name: 'answer', type: 'string', required: true, description: "the answer, grounded in context; say \"I don't know\" if context doesn't cover it" }, + { name: 'citations', type: 'object', required: false, description: 'array of { source } the answer relied on' }, + ], +}; + +const answererPrompt = (i: { question: string; context: string }) => [ + 'You are a support assistant. Answer the user using ONLY the knowledge-base context below.', + "If the context does not contain the answer, say you don't know and suggest where to look — do not guess.", + 'For each claim in your answer, cite the source it came from.', + '', + `Context:\n${i.context}`, + '', + `Question: ${i.question}`, + 'Answer:', +].join('\n'); + +/** ChainOfThought answerer — this is the module you tune with MIPROv2/GEPA. */ +export const answerer = new ChainOfThought({ + name: 'SupportAnswerer', + signature: answererSignature, + promptTemplate: answererPrompt, +}); + +/** The full appliance: Retrieve (MMR) → ChainOfThought. `run({ question })` → { answer, citations, passages, context }. */ +export async function buildSupportBot(opts?: { k?: number; mmrLambda?: number }): Promise { + const kb = await openKB(); + const retrieve = new RetrieveModule({ + client: kb, + k: opts?.k ?? 4, + useMMR: true, + mmrLambda: opts?.mmrLambda ?? 0.5, + overFetchFactor: 3, + textField: 'text', + }); + // Pipeline step 1: { question } -> { question, context } (retrieve) + // Pipeline step 2: { question, context } -> { answer, citations } (answerer) + return new Pipeline([ + { module: retrieve, map: (i: { question: string }) => ({ query: i.question }), merge: (i, o: { context: string }) => ({ ...i, context: o.context }) }, + { module: answerer }, + ] as any); +} + +/** Quality metric for tuning: helpfulness × groundedness, with "I don't know" treated as correct when the KB lacks the answer. */ +export const supportMetric: MetricFunction = ( + input: { question: string; context: string }, + out: { answer?: string; citations?: { source: string }[] }, + gold?: { answer?: string; answerable?: boolean }, +) => { + if (!out?.answer) return 0; + const said = out.answer.toLowerCase(); + const punted = /\b(i (don'?t|do not) know|not (sure|covered)|isn'?t in|no information)\b/.test(said); + // honesty: if the KB can't answer (gold.answerable === false), punting IS the right answer + if (gold && gold.answerable === false) return punted ? 1 : 0.1; + // helpfulness + let q: number; + if (gold?.answer) { + const g = gold.answer.trim().toLowerCase(); + q = said === g ? 1 : said.includes(g) ? 0.6 : 0.25; + } else { + q = punted ? 0.2 : 0.5; // no gold: a confident answer is plausibly fine, a punt is weak + } + // groundedness: cited sources should appear in the retrieved context + const cited = out.citations ?? []; + const f = cited.length === 0 ? 0.4 + : cited.every((c) => input.context.includes(c.source)) ? 1.0 + : 0.2; // hallucinated citation + return 0.6 * q + 0.4 * f; +}; + +/** Shape of the Q/A set you tune against. answerable:false ⇒ the KB intentionally doesn't cover it (tests honesty). */ +export type SupportExample = TrainingExample<{ question: string }, { answer?: string; answerable?: boolean }>; diff --git a/plugins/dspy-core/.claude-plugin/plugin.json b/plugins/dspy-core/.claude-plugin/plugin.json new file mode 100644 index 0000000..3b4d5c1 --- /dev/null +++ b/plugins/dspy-core/.claude-plugin/plugin.json @@ -0,0 +1,28 @@ +{ + "name": "dspy-core", + "description": "Scaffold, compile and evaluate DSPy.ts programs from Claude Code / Codex. Wraps dspy.ts@^2.2.0 — signatures, modules (Predict/ChainOfThought/ReAct/Retrieve/Pipeline), optimizers (BootstrapFewShot/MIPROv2/GEPA) and the AgentDB memory layer.", + "version": "0.1.0", + "author": { + "name": "rUv", + "url": "https://github.com/ruvnet" + }, + "homepage": "https://github.com/ruvnet/dspy.ts", + "license": "MIT", + "keywords": [ + "dspy", + "dspy.ts", + "agentdb", + "prompt-optimization", + "miprov2", + "gepa", + "rag" + ], + "mcpServers": { + "dspy-core": { + "command": "node", + "args": [ + "${CLAUDE_PLUGIN_ROOT}/mcp/server.js" + ] + } + } +} diff --git a/plugins/dspy-core/agents/dspy-architect.md b/plugins/dspy-core/agents/dspy-architect.md new file mode 100644 index 0000000..c950419 --- /dev/null +++ b/plugins/dspy-core/agents/dspy-architect.md @@ -0,0 +1,14 @@ +--- +name: dspy-architect +description: Designs DSPy.ts programs — picks the right module (Predict / ChainOfThought / ReAct / Retrieve / Pipeline), writes typed Signatures, designs metrics, and chooses an optimizer. Use when turning a task into a DSPy.ts program. +--- +You are the DSPy.ts architect. Given a task, produce a concrete DSPy.ts design. + +DECIDE: +1. **Module** — `PredictModule` (single-step), `ChainOfThought` (needs reasoning), `ReAct` (needs tools/lookup; add `ReActReflexion` if it should learn from failures), `RetrieveModule` (RAG; usually `Retrieve → ChainOfThought` in a `Pipeline`), or a `Pipeline` of several. +2. **Signature** — typed `inputs`/`outputs` (`{ name, type: 'string'|'number'|'boolean'|'object', required }`). Keep names semantic; one output per thing you want to score. +3. **Metric** — `(input, output, gold?) => number` in [0,1]. Reward partial correctness; never 0/1 only (optimizers need gradient). For grounded tasks, also score citation/faithfulness. +4. **Optimizer** — `BootstrapFewShot` (cheap, few-shot), `MIPROv2` (instructions + demos, `auto='light'`-style search; add `replayStore` for cross-run warm-start), `GEPA` (reflective Pareto evolution; best when you have a good metric and want the prompt to *evolve*). Add a `CompilationTracer` so trials are inspectable. +5. **Memory** — if it should remember: an `AgentDBClient` (HNSW + RaBitQ + tiers) for RAG / experience replay / reflexion. + +DELIVER: the `Signature`, the module instantiation, the `metric`, the optimizer config, a 5-10 example trainset shape, and the file layout. Then hand off to `/dspy-compile`. diff --git a/plugins/dspy-core/commands/dspy-compile.md b/plugins/dspy-core/commands/dspy-compile.md new file mode 100644 index 0000000..9ff776f --- /dev/null +++ b/plugins/dspy-core/commands/dspy-compile.md @@ -0,0 +1,15 @@ +--- +description: Optimize a DSPy.ts program with an optimizer (BootstrapFewShot / MIPROv2 / GEPA) against its metric + a trainset, and save the optimized program. +argument-hint: " [optimizer: bootstrap|mipro|gepa] [trainset: path/to/data.json]" +--- +Compile (optimize) the DSPy.ts program at `src/dspy/.ts`. Parse `$ARGUMENTS` for the program name, optimizer (default `mipro`), and a trainset JSON path (`[{ input, output? }]`). + +1. Load the program + its `metric` from `src/dspy/.ts`; load the trainset. +2. Build the optimizer: + - `bootstrap` → `new BootstrapFewShot(metric, { maxLabeledDemos, maxBootstrappedDemos })` + - `mipro` → `new MIPROv2(metric, { numTrials, numCandidateInstructions, replayStore?, tracer? })` + - `gepa` → `new GEPA(metric, { numIterations, mutationsPerStep, frontierStore? })` + (offer to wire an `AgentDBClient` for `replayStore` / `frontierStore` so the search warm-starts across runs, and a `CompilationTracer` for the trial trace.) +3. `const compiled = await optimizer.compile(program, trainset);` +4. `optimizer.save('src/dspy/.optimized.json')`; report the best score / trial trace (`optimizer.result`). +5. Show how to load it: `optimizer.load('src/dspy/.optimized.json')`. diff --git a/plugins/dspy-core/commands/dspy-eval.md b/plugins/dspy-core/commands/dspy-eval.md new file mode 100644 index 0000000..30283c5 --- /dev/null +++ b/plugins/dspy-core/commands/dspy-eval.md @@ -0,0 +1,10 @@ +--- +description: Evaluate a DSPy.ts program (raw or optimized) against its metric over a dataset, and report per-example + aggregate scores. +argument-hint: " [dataset: path/to/eval.json] [--optimized]" +--- +Evaluate `src/dspy/.ts` (or its `.optimized.json` if `--optimized`) over the eval dataset (`[{ input, output }]`) using the program's `metric`. + +1. Load the program (or `optimizer.load(...)` the optimized one) + `metric` + the dataset. +2. For each example: `const out = await program.run(ex.input); scores.push(metric(ex.input, out, ex.output));` +3. Report: mean score, min/max, the worst N examples (input + got + expected + score) — these are the candidates a `gepa` reflect step would target. +4. If both raw and optimized exist, run both and print the delta. diff --git a/plugins/dspy-core/commands/dspy-new.md b/plugins/dspy-core/commands/dspy-new.md new file mode 100644 index 0000000..3c810ec --- /dev/null +++ b/plugins/dspy-core/commands/dspy-new.md @@ -0,0 +1,16 @@ +--- +description: Scaffold a new DSPy.ts program — a typed Signature wrapped in a module (Predict / ChainOfThought / ReAct / Retrieve) and a metric stub. +argument-hint: " [signature: in1,in2 -> out1,out2] [module: predict|cot|react|retrieve]" +--- +You are scaffolding a DSPy.ts program. Parse `$ARGUMENTS` for a name, an optional `in -> out` signature, and an optional module type (default `cot`). + +1. Read the repo's `package.json` to confirm `dspy.ts` is a dependency (suggest `npm i dspy.ts` if not). +2. Create `src/dspy/.ts`: + - import the right module + `configureLM` from `'dspy.ts'` + - define a `Signature` (inputs/outputs from the parsed spec, each `{ name, type, required: true }`) + - instantiate the module with a `promptTemplate` + - export the module and a `metric(input, output, gold?) => number` stub +3. Create `src/dspy/.spec.ts` — a Jest/Vitest test that `configureLM(new DummyLM())`, runs the module, asserts the output shape. +4. Print next steps: configure a real LM, write the metric, then `/dspy-compile `. + +Keep it minimal and idiomatic — match the surrounding code style. diff --git a/plugins/dspy-core/mcp/server.js b/plugins/dspy-core/mcp/server.js new file mode 100644 index 0000000..cef2905 --- /dev/null +++ b/plugins/dspy-core/mcp/server.js @@ -0,0 +1,36 @@ +#!/usr/bin/env node +/** + * dspy-core MCP server — exposes DSPy.ts as MCP tools + resources so Claude Code + * / Codex can compile, evaluate, and run DSPy.ts programs and read the docs. + * + * Tools (stdio): dspy_compile (run an optimizer on a program), dspy_eval + * (score a program over a dataset), dspy_retrieve (RAG over an AgentDB store), + * dspy_scaffold (emit a program skeleton). + * Resources: dspy://docs/api, dspy://examples, dspy://signature-design, + * dspy://metric-design. + * + * This is a scaffold: the tool handlers shell out to `npx ts-node` against the + * project's `src/dspy/*` and to `dspy.ts`. Flesh them out per the plugin's + * commands. Run: `node plugins/dspy-core/mcp/server.js` (registered via the + * plugin's `mcpServers` config). + */ +'use strict'; +// Minimal MCP stdio scaffold — replace with @modelcontextprotocol/sdk wiring. +const TOOLS = [ + { name: 'dspy_scaffold', description: 'Emit a DSPy.ts program skeleton for a signature spec ("in1,in2 -> out1") and module type.', inputSchema: { type: 'object', properties: { name: { type: 'string' }, signature: { type: 'string' }, module: { type: 'string', enum: ['predict', 'cot', 'react', 'retrieve'] } }, required: ['name', 'signature'] } }, + { name: 'dspy_compile', description: 'Optimize a DSPy.ts program (BootstrapFewShot | MIPROv2 | GEPA) against its metric + a trainset; returns the best score + trial trace.', inputSchema: { type: 'object', properties: { program: { type: 'string' }, optimizer: { type: 'string', enum: ['bootstrap', 'mipro', 'gepa'] }, trainset: { type: 'string', description: 'path to [{input,output?}] JSON' } }, required: ['program', 'trainset'] } }, + { name: 'dspy_eval', description: 'Evaluate a DSPy.ts program (raw or optimized) over a dataset using its metric; returns per-example + aggregate scores.', inputSchema: { type: 'object', properties: { program: { type: 'string' }, dataset: { type: 'string' }, optimized: { type: 'boolean' } }, required: ['program', 'dataset'] } }, + { name: 'dspy_retrieve', description: 'RAG: embed a query and retrieve the top-k passages from an AgentDB store (HNSW + MMR).', inputSchema: { type: 'object', properties: { storePath: { type: 'string' }, query: { type: 'string' }, k: { type: 'number' } }, required: ['query'] } }, +]; +const RESOURCES = [ + { uri: 'dspy://docs/api', name: 'DSPy.ts API reference', description: 'TypeDoc output (run `npm run docs`).', mimeType: 'text/markdown' }, + { uri: 'dspy://examples', name: 'DSPy.ts examples', description: 'Runnable examples — classification, CoT, ReAct, MIPROv2, GEPA, retrieve.', mimeType: 'text/markdown' }, + { uri: 'dspy://signature-design', name: 'Signature design guide', mimeType: 'text/markdown' }, + { uri: 'dspy://metric-design', name: 'Metric design guide', mimeType: 'text/markdown' }, +]; +module.exports = { TOOLS, RESOURCES }; +// TODO: wire @modelcontextprotocol/sdk StdioServerTransport; implement handlers +// that call `npx ts-node` against src/dspy/* and dspy.ts optimizers. +if (require.main === module) { + process.stderr.write('[dspy-core mcp] scaffold — implement the MCP stdio transport + handlers. Tools: ' + TOOLS.map(t => t.name).join(', ') + '\n'); +} diff --git a/plugins/dspy-core/skills/metric-design/SKILL.md b/plugins/dspy-core/skills/metric-design/SKILL.md new file mode 100644 index 0000000..bb5ff59 --- /dev/null +++ b/plugins/dspy-core/skills/metric-design/SKILL.md @@ -0,0 +1,29 @@ +--- +name: metric-design +version: "0.1.0" +author: rUv +tags: [dspy, metric, evaluation, optimizer] +description: > + How to write metrics for DSPy.ts optimizers — `(input, output, gold?) => number` in [0,1] that gives the search a gradient. + Use when: setting up BootstrapFewShot / MIPROv2 / GEPA, or when a compile isn't improving. +--- +# Metric Design + +`type MetricFunction = (input, output, expected?) => number` — return a score in **[0, 1]**. + +## Rules of thumb +- **Never 0-or-1 only.** A binary metric gives the optimizer no gradient — most candidates score 0, the search wanders. Reward partial correctness (e.g., `0.3` for "answered, wrong"; `1` for "exact"; in between for "close"). +- **Be cheap and deterministic** where possible. The metric runs once per eval example per trial (`numTrials × |trainset|` for MIPROv2). Heavy LM-judge metrics 10× the cost — use them sparingly or on a minibatch (`MIPROv2`'s `minibatchSize`). +- **Score what you care about.** Exact match for closed QA; F1/overlap for extraction; for grounded answers, *also* score citation/faithfulness so the optimizer can't game it by being vague. +- **Use `expected`** when you have gold labels; fall back to a heuristic when you don't (the optimizer still bootstraps demos from unlabeled examples that score high on the heuristic). +- **GEPA needs a good metric most of all** — its reflect step targets the lowest-scoring examples, so a noisy metric sends it chasing noise. + +## Example +```ts +const metric = (_in: { question: string }, out: { answer: string }, gold?: { answer: string }) => { + if (!out?.answer) return 0; + if (gold && out.answer.trim().toLowerCase() === gold.answer.trim().toLowerCase()) return 1; + if (gold && out.answer.toLowerCase().includes(gold.answer.toLowerCase())) return 0.6; // partial + return 0.3; // answered, but not matching gold +}; +``` diff --git a/plugins/dspy-core/skills/signature-design/SKILL.md b/plugins/dspy-core/skills/signature-design/SKILL.md new file mode 100644 index 0000000..fcabeeb --- /dev/null +++ b/plugins/dspy-core/skills/signature-design/SKILL.md @@ -0,0 +1,29 @@ +--- +name: signature-design +version: "0.1.0" +author: rUv +tags: [dspy, signature, schema] +description: > + How to design good DSPy.ts Signatures — typed input/output specs that modules and optimizers build prompts from. + Use when: defining or refactoring a DSPy.ts module's signature. +--- +# Signature Design + +A `Signature` is `{ inputs: FieldDefinition[], outputs: FieldDefinition[] }`, each field `{ name, type: 'string'|'number'|'boolean'|'object', required?, description? }`. + +## Rules of thumb +- **One output per thing you'll score.** If the metric checks an answer *and* a confidence, make them two outputs — don't bury both in one string. +- **Names are part of the prompt.** `inputs: [{ name: 'question' }]` → the module renders `question: ...`. Pick names the model will understand (`context`, `evidence`, `claim`, not `x1`). +- **`description` is documentation for the model.** Use it to constrain (`"the answer, in <= 5 words"`). +- **Types are validated at runtime** (`Module.validateInput/validateOutput`). `object` covers arrays. ReAct adds `reasoning: string` and `steps: object[]` automatically. +- **Keep it small.** A 12-field signature optimizes badly. Split into a `Pipeline` of focused modules instead. + +## Examples +```ts +// QA +{ inputs: [{ name: 'question', type: 'string', required: true }], + outputs: [{ name: 'answer', type: 'string', required: true }] } +// RAG step (fed by RetrieveModule's `context`) +{ inputs: [{ name: 'question', type: 'string', required: true }, { name: 'context', type: 'string', required: true }], + outputs: [{ name: 'answer', type: 'string', required: true }, { name: 'citations', type: 'object', required: false }] } +``` diff --git a/plugins/dspy-evolution/.claude-plugin/plugin.json b/plugins/dspy-evolution/.claude-plugin/plugin.json new file mode 100644 index 0000000..a833b3b --- /dev/null +++ b/plugins/dspy-evolution/.claude-plugin/plugin.json @@ -0,0 +1,10 @@ +{ + "name": "dspy-evolution", + "description": "Exotic: GEPA-driven self-evolution for DSPy.ts programs. Run multi-generation reflective Pareto evolution against a held-out benchmark — each generation reflects on the weakest cases, mutates the program's instructions (and, optionally, swaps modules / restructures the Pipeline), persists the non-dominated frontier to AgentDB, and resumes from it next run. Includes the `evolution-coordinator` agent and MCP tools to evolve / benchmark / inspect frontiers.", + "version": "0.1.0", + "author": { "name": "rUv", "url": "https://github.com/ruvnet" }, + "homepage": "https://github.com/ruvnet/dspy.ts", + "license": "MIT", + "keywords": ["dspy", "dspy.ts", "gepa", "self-evolution", "pareto", "prompt-evolution", "benchmark", "agentdb", "exotic"], + "mcpServers": { "dspy-evolution": { "command": "node", "args": ["${CLAUDE_PLUGIN_ROOT}/mcp/server.js"] } } +} diff --git a/plugins/dspy-evolution/AGENTS.md b/plugins/dspy-evolution/AGENTS.md new file mode 100644 index 0000000..f5dad73 --- /dev/null +++ b/plugins/dspy-evolution/AGENTS.md @@ -0,0 +1,13 @@ +# dspy-evolution — for Codex / MCP clients (exotic) + +This plugin ships an MCP server (`mcp/server.js`, command `node ${CLAUDE_PLUGIN_ROOT}/mcp/server.js`) for multi-generation GEPA self-evolution of DSPy.ts programs. It's the exotic, expensive tool — reach for it only when a single `/dspy-mipro`/`/dspy-gepa` compile has plateaued. + +**Tools** +- `dspy_evolve` — run an N-generation GEPA evolution against a *held-out* benchmark: each generation reflects on the weakest cases, mutates instructions, keeps the Pareto frontier (persisted to AgentDB → warm-starts next run), optionally explores structural variants (`ChainOfThought → ReAct`, prepend `RetrieveModule`, split into a `Pipeline`). Always pass `cachePath` — evolution makes thousands of near-identical prompts. +- `dspy_benchmark_run` — score a program (raw/optimized/evolved) over a held-out dataset using its metric; mean + worst-N + variant deltas; records the result. +- `dspy_evolution_status` — the learning curve across generations, the reflections (`{from, weakExamples, mutated}`) that drove each mutation, warm-start lineage, a generation's causalChain. +- `dspy_frontier_inspect` — the current non-dominated frontier (candidates + per-example scores) to pick a deployable alternative. + +**Resources**: `dspy://benchmark-design`, `dspy://self-evolution-loop`, `dspy://evolution/{path}/frontier`. + +Handlers shell out to `npx ts-node` against `src/dspy/*` and `dspy.ts` (GEPA, CachingLM, CompilationTracer, AgentDBClient). Slash commands `/dspy-evolve`, `/dspy-benchmark`, `/dspy-evolution-status` wrap the same flows; the `evolution-coordinator` agent runs full campaigns — designs the benchmark, sets generations, decides when to explore structure, reads the curve, decides when to stop. Builds on `dspy-optimize` (GEPA) and `dspy-observability` (tracer, cache). diff --git a/plugins/dspy-evolution/agents/evolution-coordinator.md b/plugins/dspy-evolution/agents/evolution-coordinator.md new file mode 100644 index 0000000..904b695 --- /dev/null +++ b/plugins/dspy-evolution/agents/evolution-coordinator.md @@ -0,0 +1,20 @@ +--- +name: evolution-coordinator +description: Runs DSPy.ts self-evolution campaigns — designs the held-out benchmark + metric, configures multi-generation GEPA with a persistent AgentDB frontier and an LM cache, decides when to explore structural variants (module swaps, inserting Retrieve), reads the learning curve and reflections, and decides when to stop. Use to evolve a DSPy.ts program over many generations rather than a single compile. +--- +You coordinate self-evolution of DSPy.ts programs. This is the exotic, expensive end — only worth it when a single `/dspy-mipro` or `/dspy-gepa` compile has plateaued and the task genuinely needs the prompt to keep improving. + +SET UP: +1. **Benchmark** — a *held-out* dataset (never the slice GEPA reflects on) + a *graded* `metric` (binary metrics make evolution flail; see the `benchmark-design` skill). Record a `--baseline` first so generations have something to beat. +2. **Cache, always** — wrap the LM in `CachingLM` (`embed: 'model'`, `similarityThreshold ~0.985`) backed by a persistent AgentDB store. Evolution emits thousands of prompts that differ only in the instruction prefix; without the cache it's pointlessly expensive. Watch `lm.stats.hitRate`. +3. **GEPA per generation** — `new GEPA(metric, { numIterations, mutationsPerStep, frontierSize, frontierStore: persistentStore, seed: 42+g })`. The frontier *persists in the store*, so generation `g>1` warm-starts (`result.warmStarted === true`) and keeps evolving the non-dominated set. Carry the best program forward into the next generation. +4. **Tracer** — share the same store with a `CompilationTracer`; you get a `causalChain` per generation. +5. **Structural exploration (optional, exotic)** — between generations, propose a variant of the program's *structure*, not just its prompt: `ChainOfThought → ReAct` (when the task needs tools/lookup), prepend a `RetrieveModule` (when answers should be grounded), split a fat module into a `Pipeline`. Let the next generation's GEPA evolve the variant; keep it only if it *dominates* the incumbent on the benchmark. Don't churn structure every generation — try a variant only when the curve has stalled. + +READ THE RUN: +- **Learning curve** (`/dspy-evolution-status --curve`) — should rise then plateau. If it's flat from the start, the benchmark/metric is the problem, not GEPA. +- **Reflections** (`--reflections`) — `{ from → weakExamples → mutated }` per generation: what it decided was wrong and how it changed the instruction. This is your audit trail. +- **Frontier** (`--frontier`) — pick the deployable candidate; the highest mean isn't always the right one (check sub-slice scores). +- **Stop** when the best meanScore hasn't moved for ~2 generations *and* a structural variant didn't help. Re-runnable later from the same store. + +DELIVER: the benchmark + baseline, the evolved program (`.evolved.json`), the learning curve, the chosen frontier candidate with a one-paragraph rationale from the reflections, and the cache hit-rate. diff --git a/plugins/dspy-evolution/commands/dspy-benchmark.md b/plugins/dspy-evolution/commands/dspy-benchmark.md new file mode 100644 index 0000000..3c0468c --- /dev/null +++ b/plugins/dspy-evolution/commands/dspy-benchmark.md @@ -0,0 +1,11 @@ +--- +description: Define and run a benchmark for a DSPy.ts program — a held-out dataset + metric — scoring the raw program (and any evolved/optimized variants), and recording the result so /dspy-evolve can chart progress across generations. +argument-hint: " [--variant raw|optimized|evolved|all] [--store path/to/agentdb] [--baseline]" +--- +Score `src/dspy/.ts` (and/or its `.optimized.json` / `.evolved.json` variants) over a held-out dataset using the program's `metric`. Parse `$ARGUMENTS` for the program, the dataset (`[{input, output}]`), `--variant` (default `all`), `--store` (AgentDB path to record results into; default `.dspy/evolution`), `--baseline` (mark this run as the baseline to compare future generations against). + +1. Load the dataset + `metric`. The benchmark set must be **held out** — never the trainset GEPA reflected on, or you're scoring memorization. +2. For each requested variant: load it (`opt.load(...)` for optimized/evolved; the bare module for raw), run it over the dataset, collect per-example scores. +3. Report: mean (the headline), min/max, the worst N examples (input · got · expected · score) — these are exactly what the next generation's GEPA reflect step will target. If multiple variants, the deltas (`raw → optimized → evolved`). +4. Persist a `benchmark-result` record to `--store` (`hashEmbed`-keyed): `{ program, variant, mean, n, ts, baseline? }`. `/dspy-evolve` and `/dspy-evolution-status` read these to draw the learning curve. +5. A flat curve across generations almost always means the **benchmark or the metric** is the bottleneck (too easy, too noisy, or not measuring what you care about) — not "GEPA can't do better". Fix the benchmark first. diff --git a/plugins/dspy-evolution/commands/dspy-evolution-status.md b/plugins/dspy-evolution/commands/dspy-evolution-status.md new file mode 100644 index 0000000..54399e1 --- /dev/null +++ b/plugins/dspy-evolution/commands/dspy-evolution-status.md @@ -0,0 +1,11 @@ +--- +description: Inspect a DSPy.ts evolution store — the learning curve across generations, the current Pareto frontier, the reflections that drove each mutation, and warm-start lineage. +argument-hint: " [--curve] [--frontier] [--reflections] [--gen N]" +--- +Open the `AgentDBClient` evolution store at `` and report what `/dspy-evolve` has accumulated. Parse `$ARGUMENTS` for the store path, optional `--gen N`, and which views (default: all). + +1. `const store = new AgentDBClient({ vectorDimension: 64, storage: { path } }); await store.init(); const tracer = new CompilationTracer({ store });` +2. `--curve` — the per-generation best `meanScore` from the `gen-result` / `benchmark-result` records: the learning curve. Flat ⇒ benchmark/metric problem (see the `benchmark-design` skill). +3. `--frontier` — the current non-dominated `gepa-candidate` set: each `{ instruction, meanScore, perExampleScores }`. The user can pick a different non-dominated candidate (e.g. one that's slightly lower mean but stronger on a sub-slice they care about). +4. `--reflections` — for `--gen N` (or all): the GEPA reflections — `{ from, weakExamples, mutated }` — i.e. *why* each instruction was changed. This is the "what did it learn and how" trail. +5. `--gen N` also shows that generation's `causalChain` via the tracer (which trials led to that gen's best). `store.getStats()` / `tierCounts()` for size. diff --git a/plugins/dspy-evolution/commands/dspy-evolve.md b/plugins/dspy-evolution/commands/dspy-evolve.md new file mode 100644 index 0000000..c4f7c01 --- /dev/null +++ b/plugins/dspy-evolution/commands/dspy-evolve.md @@ -0,0 +1,17 @@ +--- +description: Run a multi-generation GEPA self-evolution of a DSPy.ts program against a held-out benchmark — reflect on the weakest cases, mutate instructions, keep the Pareto frontier, persist it to AgentDB, and resume next time. +argument-hint: " [--generations N] [--iterations-per-gen N] [--mutations N] [--frontier N] [--store path/to/agentdb] [--cache path/to/agentdb] [--explore-modules]" +--- +Evolve `src/dspy/.ts`. Parse `$ARGUMENTS` for the program, the benchmark dataset (`[{input, output?}]` — see the `benchmark-design` skill), `--generations` (default 5), `--iterations-per-gen` (GEPA iterations within a generation, default 10), `--mutations` (per step, default 2), `--frontier` (frontier size, default 8), `--store` (AgentDB path → the persistent `frontierStore`; default `.dspy/evolution`), `--cache` (an AgentDB path → wrap the LM in `CachingLM` — strongly recommended, evolution makes thousands of near-identical prompts), `--explore-modules` (also try swapping the answerer module / inserting a Retrieve step between generations — exotic). + +1. Set up: `const store = new AgentDBClient({ vectorDimension: 64, storage: { path: storePath } }); await store.init();` + if `--cache`: `const cache = new AgentDBClient({ vectorDimension: 384, storage: { path: cachePath } }); await cache.init(); configureLM(new CachingLM(getLM(), { store: cache, similarityThreshold: 0.985, embed: 'model' }));` + `const tracer = new CompilationTracer({ store });` +2. Load the program + its `metric` + the benchmark; split a fixed `eval` slice. +3. For `g` in `1..generations`: + - `const gepa = new GEPA(metric, { numIterations: iterationsPerGen, mutationsPerStep, frontierSize, frontierStore: store, seed: 42 + g });` + - `const evolved = await gepa.compile(programForGen, evalSlice);` — `g > 1` ⇒ `gepa.result.warmStarted === true` (the frontier persists in `store`). + - Record generation `g`'s best `meanScore` (and the frontier) via the tracer / a `gen-result` record. If `--explore-modules`, between generations propose a structural variant (e.g. `ChainOfThought → ReAct`, or prepend a `RetrieveModule`) and let the next generation's GEPA evolve *that* too — keep it only if it dominates. + - `programForGen = gepa.compiledProgram` (carry the best forward). +4. Stop early if the best `meanScore` hasn't improved for 2 generations (the frontier has converged). +5. `gepa.save('src/dspy/.evolved.json');` Report: per-generation best scores (the learning curve), the final frontier (instruction + meanScore, so the user can pick a non-dominated alternative), `result.warmStarted`, total LM calls (and cache `hitRate`). Re-run with the same `--store` ⇒ evolution continues from where it stopped. diff --git a/plugins/dspy-evolution/mcp/server.js b/plugins/dspy-evolution/mcp/server.js new file mode 100644 index 0000000..f31d50c --- /dev/null +++ b/plugins/dspy-evolution/mcp/server.js @@ -0,0 +1,32 @@ +#!/usr/bin/env node +/** + * dspy-evolution MCP server — exotic: multi-generation GEPA self-evolution for DSPy.ts. + * Tools: dspy_evolve (run an N-generation GEPA evolution against a held-out benchmark, + * persistent AgentDB frontier, optional LM cache, optional structural exploration), + * dspy_benchmark_run (score a program/variant over a held-out dataset, record the result), + * dspy_evolution_status (learning curve / reflections / warm-start lineage from an evolution store), + * dspy_frontier_inspect (the current Pareto frontier: candidates with per-example scores). + * Resources: dspy://benchmark-design, dspy://self-evolution-loop, dspy://evolution/{path}/frontier. + * + * Scaffold: handlers shell out to `npx ts-node` against src/dspy/* and dspy.ts + * (GEPA, CachingLM, CompilationTracer, AgentDBClient, RetrieveModule/ReAct for variants). + * Flesh out the @modelcontextprotocol/sdk stdio wiring + handlers. + */ +'use strict'; +const TOOLS = [ + { name: 'dspy_evolve', description: 'Run a multi-generation GEPA self-evolution of a DSPy.ts program against a held-out benchmark: each generation reflects on the weakest cases, mutates instructions, keeps the Pareto frontier (persisted to AgentDB → warm-starts next run), optionally explores structural variants (module swap / insert Retrieve). Returns {generations:[{best, meanScore}], frontier, warmStarted, lmCalls, cacheHitRate}.', inputSchema: { type: 'object', properties: { program: { type: 'string' }, benchmark: { type: 'string', description: 'held-out [{input,output?}] JSON' }, generations: { type: 'number' }, iterationsPerGen: { type: 'number' }, mutationsPerStep: { type: 'number' }, frontierSize: { type: 'number' }, storePath: { type: 'string', description: 'AgentDB path for the persistent frontier' }, cachePath: { type: 'string', description: 'AgentDB path for the CachingLM (recommended)' }, exploreModules: { type: 'boolean', description: 'also try structural variants between generations (exotic)' } }, required: ['program', 'benchmark'] } }, + { name: 'dspy_benchmark_run', description: 'Score a DSPy.ts program (raw / optimized / evolved) over a HELD-OUT dataset using its metric; reports mean, min/max, the worst N examples, and (if multiple) variant deltas; records a benchmark-result to the evolution store.', inputSchema: { type: 'object', properties: { program: { type: 'string' }, dataset: { type: 'string' }, variant: { type: 'string', enum: ['raw', 'optimized', 'evolved', 'all'] }, storePath: { type: 'string' }, baseline: { type: 'boolean' } }, required: ['program', 'dataset'] } }, + { name: 'dspy_evolution_status', description: 'Inspect an evolution store: the per-generation learning curve, the GEPA reflections that drove each mutation ({from, weakExamples, mutated}), warm-start lineage, and a generation’s causalChain.', inputSchema: { type: 'object', properties: { storePath: { type: 'string' }, gen: { type: 'number' }, view: { type: 'string', enum: ['curve', 'reflections', 'all'] } }, required: ['storePath'] } }, + { name: 'dspy_frontier_inspect', description: 'The current non-dominated Pareto frontier in an evolution store: each candidate {instruction, meanScore, perExampleScores} so a caller can pick a deployable alternative.', inputSchema: { type: 'object', properties: { storePath: { type: 'string' } }, required: ['storePath'] } }, +]; +const RESOURCES = [ + { uri: 'dspy://benchmark-design', name: 'Benchmark design guide', description: 'Designing the held-out benchmark + metric a self-evolution loop optimizes against.', mimeType: 'text/markdown' }, + { uri: 'dspy://self-evolution-loop', name: 'Self-evolution loop guide', description: 'Multi-generation GEPA, persistent frontier, warm-start, caching, structural exploration.', mimeType: 'text/markdown' }, + { uri: 'dspy://evolution/{path}/frontier', name: 'Evolution frontier', description: 'Live Pareto frontier + learning curve for an evolution store.', mimeType: 'application/json' }, +]; +module.exports = { TOOLS, RESOURCES }; +// TODO: wire @modelcontextprotocol/sdk StdioServerTransport; handlers shell out to +// `npx ts-node` against src/dspy/* and dspy.ts (GEPA, CachingLM, CompilationTracer, AgentDBClient, RetrieveModule/ReAct). +if (require.main === module) { + process.stderr.write('[dspy-evolution mcp] scaffold — handlers shell out to `npx ts-node` against src/dspy/* and dspy.ts. Tools: ' + TOOLS.map(t => t.name).join(', ') + '\n'); +} diff --git a/plugins/dspy-evolution/skills/benchmark-design/SKILL.md b/plugins/dspy-evolution/skills/benchmark-design/SKILL.md new file mode 100644 index 0000000..b772822 --- /dev/null +++ b/plugins/dspy-evolution/skills/benchmark-design/SKILL.md @@ -0,0 +1,27 @@ +--- +name: benchmark-design +version: "0.1.0" +author: rUv +tags: [dspy, evolution, benchmark, evaluation, gepa, metric] +description: > + How to design the held-out benchmark + metric that a DSPy.ts self-evolution loop optimizes against — coverage, difficulty spread, leakage, and why a bad benchmark makes GEPA flail. + Use when: setting up `/dspy-benchmark` / `/dspy-evolve`, or when the evolution learning curve is flat. +--- +# Designing an evolution benchmark + +GEPA reflects on the *lowest-scoring* examples each iteration. A multi-generation `/dspy-evolve` does that over and over. So the benchmark + metric *are* the objective — get them wrong and you'll evolve a program that's great at the wrong thing. + +## Rules of thumb +- **Held out, always.** The benchmark must be disjoint from the slice GEPA reflects/bootstraps on. If they overlap you're measuring (and rewarding) memorization. `/dspy-benchmark` and `/dspy-evolve --benchmark` should point at the held-out set; the reflection slice is separate. +- **Cover the task, not just the easy middle.** Include the edge cases, the ambiguous inputs, the "should say I don't know" cases. The frontier you get is only as broad as the benchmark; uncovered behaviors won't improve (and may regress silently). +- **Spread the difficulty.** If everything scores ~0.9 from generation 1, there's no gradient — the curve is flat because there's nothing to climb, not because GEPA failed. Mix in hard examples so early generations score in the 0.3–0.7 range. +- **Size: enough to be stable, small enough to be cheap.** Each generation evaluates the whole benchmark × GEPA's internal trials. 30–100 examples is a common sweet spot; with a `CachingLM` you can afford more. +- **The metric must be graded** (partial credit in [0,1]), reasonably cheap, deterministic where possible, and aligned with what you actually want — including penalizing confident-wrong and rewarding honest "I don't know". (See `dspy-core`'s `metric-design` and, for RAG, `dspy-rag`'s `grounding-and-citations`.) +- **Pick a baseline** (`/dspy-benchmark --baseline`) before evolving — "generation 5 scored 0.78" is meaningless without "the raw program scored 0.52". +- **Watch for Goodhart.** If the metric is gameable (e.g. rewards length, or keyword presence), evolution *will* find the exploit. Spot-check the highest-scoring frontier candidates by hand; if a high score looks wrong, the metric is wrong. + +## Diagnosing a flat learning curve +1. Are benchmark and reflection slices actually disjoint? (leakage → already-saturated scores) +2. Is the metric binary or noisy? (no gradient / chasing noise) +3. Is the benchmark too easy? (everything ~0.9 from gen 1) +4. Only after ruling those out: maybe the task is at the model's ceiling — then a *structural* variant (`--explore-modules`: add Retrieve, switch to ReAct) is the lever, not more GEPA iterations. diff --git a/plugins/dspy-evolution/skills/self-evolution-loop/SKILL.md b/plugins/dspy-evolution/skills/self-evolution-loop/SKILL.md new file mode 100644 index 0000000..f1732d9 --- /dev/null +++ b/plugins/dspy-evolution/skills/self-evolution-loop/SKILL.md @@ -0,0 +1,39 @@ +--- +name: self-evolution-loop +version: "0.1.0" +author: rUv +tags: [dspy, evolution, gepa, pareto, agentdb, exotic] +description: > + How DSPy.ts self-evolution works — multi-generation GEPA with a persistent AgentDB Pareto frontier, warm-start across runs, LM caching, and optional structural exploration. + Use when: running or designing a `/dspy-evolve` campaign, or deciding generations / when to explore structure. +--- +# The self-evolution loop + +`/dspy-evolve` runs GEPA generation after generation, each one continuing from the last via a persistent frontier in AgentDB. + +## The loop +``` +for g in 1..generations: + gepa = new GEPA(metric, { numIterations, mutationsPerStep, frontierSize, + frontierStore: persistentStore, seed: 42+g }) + evolved = await gepa.compile(programForGen, reflectionSlice) # g>1 ⇒ warmStarted + recordGeneration(g, gepa.result.best.meanScore, gepa.result.frontier) + if exploreModules and curveStalled: programForGen = proposeStructuralVariant(evolved) # exotic + else: programForGen = evolved + if best meanScore hasn't improved for 2 generations: break +gepa.save('.evolved.json') +``` +- **Frontier persistence** — `frontierStore` is an `AgentDBClient`; GEPA writes `gepa-candidate` records and, on a later compile of the same task fingerprint, rebuilds the non-dominated set from them → `result.warmStarted === true`. That's what makes "generation 6 tomorrow" pick up where generation 5 left off. +- **Reflection slice ≠ benchmark** — GEPA reflects on the weakest examples of the slice you pass it; the held-out *benchmark* (`/dspy-benchmark`) is what you score generations against. Keep them disjoint (see `benchmark-design`). +- **Cache the LM** — `CachingLM` (`embed: 'model'`, threshold ~0.985) over a persistent store. Generations produce thousands of prompts that differ only in the instruction prefix; the cache makes that affordable. Track `hitRate`. +- **Tracer** — share the store with a `CompilationTracer` for a per-generation `causalChain`. + +## Structural exploration (the exotic part) +Between generations, when the prompt-only curve has stalled, propose a change to the program's *structure*: +- `ChainOfThought → ReAct` + a tool registry — when the task needs lookup/computation the prompt can't supply. +- Prepend a `RetrieveModule` (→ a `Pipeline`) — when answers should be grounded in a corpus. +- Split a fat module into a `Pipeline` of focused ones — when one signature is doing too much. +Let the next generation's GEPA evolve the variant's instructions; **keep it only if it dominates** the incumbent on the benchmark. This is search over architectures, not just prompts — powerful, slow, easy to overfit. Try one variant at a time, only when stalled, and always re-baseline. + +## When to use this at all +Single-compile `/dspy-mipro` or `/dspy-gepa` first. Reach for `/dspy-evolve` only when that's plateaued and the task is important enough to spend many generations × an LM budget on. It's the exotic tool — most programs don't need it. diff --git a/plugins/dspy-observability/.claude-plugin/plugin.json b/plugins/dspy-observability/.claude-plugin/plugin.json new file mode 100644 index 0000000..5cb41fb --- /dev/null +++ b/plugins/dspy-observability/.claude-plugin/plugin.json @@ -0,0 +1,10 @@ +{ + "name": "dspy-observability", + "description": "Observability for DSPy.ts — CompilationTracer captures every optimizer trial with causedBy links (causalChain to the best), persisted to AgentDB with optional MLflow export; CachingLM wraps any LM with a fuzzy AgentDB vector cache (similarity-threshold hits, TTL, hit stats). Includes the `observability-engineer` agent and MCP tools for tracing/caching.", + "version": "0.1.0", + "author": { "name": "rUv", "url": "https://github.com/ruvnet" }, + "homepage": "https://github.com/ruvnet/dspy.ts", + "license": "MIT", + "keywords": ["dspy", "dspy.ts", "observability", "tracing", "compilation-tracer", "mlflow", "caching", "agentdb"], + "mcpServers": { "dspy-observability": { "command": "node", "args": ["${CLAUDE_PLUGIN_ROOT}/mcp/server.js"] } } +} diff --git a/plugins/dspy-observability/AGENTS.md b/plugins/dspy-observability/AGENTS.md new file mode 100644 index 0000000..94e550a --- /dev/null +++ b/plugins/dspy-observability/AGENTS.md @@ -0,0 +1,13 @@ +# dspy-observability — for Codex / MCP clients + +This plugin ships an MCP server (`mcp/server.js`, command `node ${CLAUDE_PLUGIN_ROOT}/mcp/server.js`) for DSPy.ts tracing and caching. + +**Tools** +- `dspy_trace` — compile a program with a `CompilationTracer` attached; trials are persisted to AgentDB with `causedBy` links. Returns `{runId, trials, causalChain, bestScore, mlflowAvailable}`. +- `dspy_runs` — list the compile runs in an AgentDB trace store (or inspect one): optimizer, params, best score, trial count, start/end. +- `dspy_run_chain` — the `causalChain` (causedBy lineage) of trials that produced the best candidate for a `runId`. +- `dspy_cache_stats` — `CachingLM` hit-rate (`{hits, misses, hitRate, entries}`) plus the backing AgentDB store stats. + +**Resources**: `dspy://compile-tracing`, `dspy://llm-caching`, `dspy://traces/{runId}`. + +Handlers shell out to `npx ts-node` against `src/dspy/*` and `dspy.ts`. Slash commands `/dspy-trace`, `/dspy-runs`, `/dspy-cache` wrap the same flows interactively; the `observability-engineer` agent attaches tracing/caching, reads traces, and explains why an optimizer landed where it did. Pairs with `dspy-optimize` (attach the tracer to `/dspy-mipro`/`/dspy-gepa`; wrap the LM with `CachingLM` before a big search). diff --git a/plugins/dspy-observability/agents/observability-engineer.md b/plugins/dspy-observability/agents/observability-engineer.md new file mode 100644 index 0000000..38ca7dc --- /dev/null +++ b/plugins/dspy-observability/agents/observability-engineer.md @@ -0,0 +1,16 @@ +--- +name: observability-engineer +description: Instruments DSPy.ts runs — attaches a CompilationTracer (AgentDB-persisted trials, causedBy links, causalChain, optional MLflow), wraps LMs with CachingLM, reads traces to explain why an optimizer landed where it did, and tunes cache thresholds. Use to make a DSPy.ts optimization legible and cheaper to re-run. +--- +You make DSPy.ts runs observable. + +WIRE: +1. **CompilationTracer** — `new CompilationTracer({ store: agentDbClient, mlflow })`. Pass it to `MIPROv2`/`GEPA`. It writes `compile-run`, `compile-trial` (each with `causedBy` = the prior trial → a lineage), and `compile-run-end` records. Read it with `getTrace(runId)` (all trials), `causalChain(runId)` (the path to the best), `runIds`/`runCount`. `mlflowAvailable` tells you if the optional `@mlflow/tracking` mirror is active. Use a persistent `storage.path` so traces outlive the process; a small `vectorDimension` (~64) is fine — these are fingerprints, not embeddings. +2. **CachingLM** — `new CachingLM(baseLM, { store, similarityThreshold, ttlMs, embed })` then `configureLM(...)`. A new prompt within `similarityThreshold` cosine of a cached one returns the stored completion. This is a big win during optimizer search (MIPROv2/GEPA generate many near-identical prompts). `embed: 'hash'` = deterministic, zero-dependency; `embed: 'model'` = embedding-service vectors (fewer false hits, costs an embed call). `lm.stats` → `{ hits, misses, hitRate, entries }`. + +READ A TRACE: +- Flat scores across the chain ⇒ the metric isn't discriminating (binary? noisy?) — fix that before adding budget. +- The best trial's lineage (`causalChain`) shows which instruction/demo changes actually moved the needle — that's where to push next. +- High cache `hitRate` during a compile is expected and good; a *low* one with many near-duplicate prompts means `similarityThreshold` is too high or hash collisions are forcing `embed: 'model'`. + +DELIVER: the instrumented compile (`/dspy-trace`), the trace readout (`/dspy-runs --chain`), the cache config + stats (`/dspy-cache --stats`), and a one-paragraph "why it landed here" from the causal chain. diff --git a/plugins/dspy-observability/commands/dspy-cache.md b/plugins/dspy-observability/commands/dspy-cache.md new file mode 100644 index 0000000..9e7a0bd --- /dev/null +++ b/plugins/dspy-observability/commands/dspy-cache.md @@ -0,0 +1,11 @@ +--- +description: Wrap a DSPy.ts LM with CachingLM — a fuzzy AgentDB vector cache that returns a stored completion when a new prompt is near-identical to a past one — and report hit-rate stats. +argument-hint: " [--threshold 0..1] [--ttl ms] [--embed hash|model] [--store path/to/agentdb] [--stats]" +--- +Add (or inspect) a `CachingLM` around the LM your program uses. Parse `$ARGUMENTS` for the program (whose `configureLM(...)` you'll wrap), `--threshold` (cosine similarity to count as a hit, default 0.97), `--ttl` (entry lifetime in ms), `--embed` (`hash` = deterministic local, `model` = embedding service), `--store` (AgentDB path; default `.dspy/llm-cache`), `--stats` (just print stats and exit). + +1. `const cache = new AgentDBClient({ vectorDimension: ${EMBED==model?384:64}, storage: { path: cachePath } }); await cache.init();` + `const lm = new CachingLM(baseLM, { store: cache, similarityThreshold: threshold, ttlMs: ttl, embed });` + `configureLM(lm);` +2. `--stats`: print `lm.stats` — `{ hits, misses, hitRate, entries }` — and the store's `getStats()` / `tierCounts()`. +3. Otherwise: note that subsequent `program.run(...)` / optimizer compiles will hit the cache for near-duplicate prompts (huge during MIPROv2/GEPA search, where many candidates differ only slightly). Lower `--threshold` → more hits but riskier (a near-but-meaningfully-different prompt reuses an answer); 0.97–0.99 is safe. Use `--embed model` if hash collisions cause false hits. diff --git a/plugins/dspy-observability/commands/dspy-runs.md b/plugins/dspy-observability/commands/dspy-runs.md new file mode 100644 index 0000000..6e1efaa --- /dev/null +++ b/plugins/dspy-observability/commands/dspy-runs.md @@ -0,0 +1,10 @@ +--- +description: List the optimizer compile runs recorded in an AgentDB trace store, and drill into one — its trials, scores, params, and the causal chain to the best candidate. +argument-hint: " [--run ] [--chain] [--limit N]" +--- +Open the `AgentDBClient` trace store at `` and report its `CompilationTracer` runs. Parse `$ARGUMENTS` for the store path, optional `--run `, `--chain` (show only the causal chain), `--limit`. + +1. `const store = new AgentDBClient({ vectorDimension: 64, storage: { path } }); await store.init(); const tracer = new CompilationTracer({ store });` +2. No `--run`: list `tracer.runIds` / `tracer.runCount` — for each, the optimizer, params, best score, trial count, start/end (`compile-run` / `compile-run-end` records). +3. `--run `: `tracer.getTrace(runId)` — every trial in order: `{ label, params, score, causedBy }`. With `--chain`: `tracer.causalChain(runId)` only — the trial lineage that produced the best. +4. `store.getStats()` for store size. This is the "what did the optimizer actually try" view — pair it with `/dspy-trace` to generate new runs. diff --git a/plugins/dspy-observability/commands/dspy-trace.md b/plugins/dspy-observability/commands/dspy-trace.md new file mode 100644 index 0000000..d5753fd --- /dev/null +++ b/plugins/dspy-observability/commands/dspy-trace.md @@ -0,0 +1,12 @@ +--- +description: Run an optimizer compile with a CompilationTracer attached, persisting every trial to AgentDB (with causedBy links), then print the run summary and the causal chain to the best candidate. +argument-hint: " [trainset: data.json] [--store path/to/agentdb] [--mlflow]" +--- +Compile `src/dspy/.ts` under a `CompilationTracer`. Parse `$ARGUMENTS` for the program, the optimizer (`mipro`/`gepa`), a trainset JSON, `--store` (an AgentDB path for the trace; default `.dspy/traces`), `--mlflow` (also mirror runs/metrics to MLflow if `@mlflow/tracking` is installed). + +1. `const store = new AgentDBClient({ vectorDimension: 64, storage: { path: tracePath } }); await store.init();` + `const tracer = new CompilationTracer({ store, mlflow });` +2. Build the optimizer with the tracer: `new MIPROv2(metric, { numTrials, tracer, replayStore: store })` or `new GEPA(metric, { numIterations, frontierStore: store /* tracer hooks too */ })`. +3. `await opt.compile(program, trainset);` +4. Print: the `runId`, `tracer.getTrace(runId)` (each `compile-trial` with `{label, params, score, causedBy}`), and `tracer.causalChain(runId)` — the path of trials leading to the best. Note `tracer.mlflowAvailable`. +5. `opt.save('src/dspy/.optimized.json');` — the saved file also carries the trial trace. Inspect later with `/dspy-runs`. diff --git a/plugins/dspy-observability/mcp/server.js b/plugins/dspy-observability/mcp/server.js new file mode 100644 index 0000000..d189802 --- /dev/null +++ b/plugins/dspy-observability/mcp/server.js @@ -0,0 +1,30 @@ +#!/usr/bin/env node +/** + * dspy-observability MCP server — exposes DSPy.ts tracing + caching as MCP tools. + * Tools: dspy_trace (compile with a CompilationTracer attached → run summary + causal chain), + * dspy_runs (list/inspect compile runs in an AgentDB trace store), dspy_run_chain + * (causalChain to the best for a runId), dspy_cache_stats (CachingLM hit-rate + store stats). + * Resources: dspy://compile-tracing, dspy://llm-caching, dspy://traces/{runId}. + * + * Scaffold: handlers shell out to `npx ts-node` against src/dspy/* and dspy.ts + * (CompilationTracer, CachingLM, MIPROv2/GEPA, AgentDBClient). Flesh out the + * @modelcontextprotocol/sdk stdio wiring + handlers. + */ +'use strict'; +const TOOLS = [ + { name: 'dspy_trace', description: 'Compile a DSPy.ts program with a CompilationTracer attached (trials persisted to AgentDB with causedBy links); returns {runId, trials, causalChain, bestScore, mlflowAvailable}.', inputSchema: { type: 'object', properties: { program: { type: 'string' }, optimizer: { type: 'string', enum: ['mipro', 'gepa'] }, trainset: { type: 'string' }, storePath: { type: 'string', description: 'AgentDB path for the trace' }, mlflow: { type: 'boolean' } }, required: ['program', 'optimizer', 'trainset'] } }, + { name: 'dspy_runs', description: 'List the optimizer compile runs in an AgentDB trace store (or inspect one): optimizer, params, best score, trial count, start/end.', inputSchema: { type: 'object', properties: { storePath: { type: 'string' }, runId: { type: 'string' }, limit: { type: 'number' } }, required: ['storePath'] } }, + { name: 'dspy_run_chain', description: 'Return the causal chain (causedBy lineage) of trials that produced the best candidate for a given runId.', inputSchema: { type: 'object', properties: { storePath: { type: 'string' }, runId: { type: 'string' } }, required: ['storePath', 'runId'] } }, + { name: 'dspy_cache_stats', description: 'CachingLM hit-rate stats {hits, misses, hitRate, entries} plus the backing AgentDB store stats (vectors, dimension, tiers, quantization).', inputSchema: { type: 'object', properties: { cachePath: { type: 'string' } }, required: ['cachePath'] } }, +]; +const RESOURCES = [ + { uri: 'dspy://compile-tracing', name: 'Compile tracing guide', description: 'Instrumenting MIPROv2/GEPA with CompilationTracer; reading getTrace / causalChain.', mimeType: 'text/markdown' }, + { uri: 'dspy://llm-caching', name: 'LLM caching guide', description: 'CachingLM threshold/TTL/embed tradeoffs and when it pays off.', mimeType: 'text/markdown' }, + { uri: 'dspy://traces/{runId}', name: 'Compilation trace', description: 'A CompilationTracer run — trials with causedBy links; causalChain to the best.', mimeType: 'application/json' }, +]; +module.exports = { TOOLS, RESOURCES }; +// TODO: wire @modelcontextprotocol/sdk StdioServerTransport; handlers shell out to +// `npx ts-node` against src/dspy/* and dspy.ts (CompilationTracer, CachingLM, MIPROv2/GEPA, AgentDBClient). +if (require.main === module) { + process.stderr.write('[dspy-observability mcp] scaffold — handlers shell out to `npx ts-node` against src/dspy/* and dspy.ts. Tools: ' + TOOLS.map(t => t.name).join(', ') + '\n'); +} diff --git a/plugins/dspy-observability/skills/compile-tracing/SKILL.md b/plugins/dspy-observability/skills/compile-tracing/SKILL.md new file mode 100644 index 0000000..429efdf --- /dev/null +++ b/plugins/dspy-observability/skills/compile-tracing/SKILL.md @@ -0,0 +1,34 @@ +--- +name: compile-tracing +version: "0.1.0" +author: rUv +tags: [dspy, observability, tracing, compilation-tracer, mlflow, agentdb] +description: > + How to instrument DSPy.ts optimizer runs with CompilationTracer — persisted trials, causedBy links, causalChain to the best, optional MLflow — and how to read the trace to understand a compile. + Use when: attaching tracing to MIPROv2 / GEPA (`/dspy-trace`, `/dspy-runs`), or debugging why a compile landed where it did. +--- +# Tracing a compile + +`CompilationTracer({ store, mlflow })` records an optimizer run into an `AgentDBClient`. + +## What it records +- `startRun(optimizer, params)` → a `runId`, and a `compile-run` record. +- `logTrial(runId, { label, params?, score })` → a `trialId`, and a `compile-trial` record whose `causedBy` is the previous trial's id — so the trials form a lineage, not just a list. +- `endRun(runId, { bestScore, ... })` → a `compile-run-end` record. +- If `mlflow: true` and `@mlflow/tracking` is installed, runs/metrics are mirrored there too (`mlflowAvailable` reports whether it engaged; absence is silently fine). + +## Reading it +- `getTrace(runId)` — every trial in order: `{ label, params, score, causedBy }`. +- `causalChain(runId)` — just the lineage of trials that led to the best candidate. This is the useful view: it shows which instruction/demo changes moved the score. +- `runIds` / `runCount` — across runs in this store. + +## Setup notes +- Use a **persistent** `storage.path` (e.g. `.dspy/traces`) so traces survive the process; the saved optimizer JSON (`opt.save(...)`) also carries the trial trace inline. +- `vectorDimension: 64` is plenty — trace records are fingerprints, not semantic embeddings. +- Share the *same* `AgentDBClient` for the tracer and the optimizer's `replayStore`/`frontierStore` — one store, correlated history. +- Records use `store.hashEmbed(...)` for their vectors (deterministic), so traces are reproducible. + +## What the trace tells you +- **Flat scores down the chain** ⇒ the metric, not the search — binary or noisy metrics give the optimizer nothing to climb. Fix `metric-design` first. +- **Where the best came from** — the chain's score jumps point at the instruction/demo edits that worked; iterate there. +- **Warm-started runs** show their recalled seeds as the first trials — you can see whether prior knowledge actually helped. diff --git a/plugins/dspy-observability/skills/llm-caching/SKILL.md b/plugins/dspy-observability/skills/llm-caching/SKILL.md new file mode 100644 index 0000000..69395b1 --- /dev/null +++ b/plugins/dspy-observability/skills/llm-caching/SKILL.md @@ -0,0 +1,27 @@ +--- +name: llm-caching +version: "0.1.0" +author: rUv +tags: [dspy, caching, caching-lm, agentdb, cost] +description: > + How to use CachingLM in DSPy.ts — a fuzzy AgentDB vector cache over an LM that returns a stored completion for near-identical prompts — including threshold/TTL/embed tradeoffs and when it matters most. + Use when: wrapping an LM with CachingLM (`/dspy-cache`), or cutting the cost/latency of an optimizer search. +--- +# CachingLM + +`new CachingLM(baseLM, { store, similarityThreshold, ttlMs, embed })` wraps any `LMDriver`. On `generate(prompt)`: embed the prompt, search the `AgentDBClient` store; if the nearest cached prompt is within `similarityThreshold` cosine, return its completion; else call `baseLM`, store the result, return it. + +## Knobs +- **`similarityThreshold`** (default ~0.97) — cosine to count as a hit. **Higher = safer, fewer hits.** 0.97–0.99: only near-identical prompts reuse an answer (good). Below ~0.95: prompts that differ in a way that *should* change the answer start reusing stale completions — only do this if you know your prompts vary trivially. +- **`ttlMs`** — entry lifetime. Use it when the "right" answer can change over time (anything time-sensitive, external state). Omit for pure functions of the prompt. +- **`embed`** — `'hash'` (default-ish): deterministic, zero extra calls, but hash-space collisions can cause false hits. `'model'`: embedding-service vectors — far fewer false hits, costs one embed call per prompt. Use `'model'` if you see wrong cache hits with `'hash'`. +- **store** — a persistent `storage.path` to keep the cache across runs; HNSW + `quantization: 'rabitq'` keep lookups fast as it grows; tiers/`evictTier` to cap size. + +## When it pays off +- **Optimizer search** — MIPROv2 and GEPA generate *many* prompts that differ only in the instruction prefix or one demo. With caching, the repeated bodies hit the cache. This is the single biggest win; always wrap the LM before a big compile. +- **Re-runs** — re-running the same eval set after a small change reuses most completions. +- **Dev loops** — iterating on a metric while the program/prompts hold steady. + +## Watch +- `lm.stats` → `{ hits, misses, hitRate, entries }`. A high `hitRate` during a compile is expected. A *low* one despite obviously-similar prompts ⇒ `similarityThreshold` too high, or switch `embed` to `'model'`. +- Caching hides nondeterminism: if `baseLM` is sampled (temperature > 0), the first answer for a prompt-cluster gets frozen. Fine for optimization (you want stability); be aware for anything that *wants* variety. diff --git a/plugins/dspy-optimize/.claude-plugin/plugin.json b/plugins/dspy-optimize/.claude-plugin/plugin.json new file mode 100644 index 0000000..294d39c --- /dev/null +++ b/plugins/dspy-optimize/.claude-plugin/plugin.json @@ -0,0 +1,10 @@ +{ + "name": "dspy-optimize", + "description": "Deep optimizer workflows for DSPy.ts — MIPROv2 with AgentDB experience replay (warm-start across runs), GEPA reflective Pareto-frontier prompt evolution, and BootstrapFewShot with input-conditioned dynamic demos. Includes the `optimizer-engineer` agent and MCP tools for compiling/replaying.", + "version": "0.1.0", + "author": { "name": "rUv", "url": "https://github.com/ruvnet" }, + "homepage": "https://github.com/ruvnet/dspy.ts", + "license": "MIT", + "keywords": ["dspy", "dspy.ts", "miprov2", "gepa", "bootstrap-fewshot", "prompt-optimization", "experience-replay", "agentdb"], + "mcpServers": { "dspy-optimize": { "command": "node", "args": ["${CLAUDE_PLUGIN_ROOT}/mcp/server.js"] } } +} diff --git a/plugins/dspy-optimize/AGENTS.md b/plugins/dspy-optimize/AGENTS.md new file mode 100644 index 0000000..dcf83a2 --- /dev/null +++ b/plugins/dspy-optimize/AGENTS.md @@ -0,0 +1,13 @@ +# dspy-optimize — for Codex / MCP clients + +This plugin ships an MCP server (`mcp/server.js`, command `node ${CLAUDE_PLUGIN_ROOT}/mcp/server.js`) that runs DSPy.ts optimizer campaigns. + +**Tools** +- `dspy_mipro` — MIPROv2: instruction proposal + demo bootstrapping + seeded search over a DSPy.ts program. Pass `replayPath` (an AgentDB dir) to warm-start later compiles of the same task; `trace: true` attaches a CompilationTracer. +- `dspy_gepa` — GEPA: reflective Pareto-frontier prompt evolution that targets the weakest examples each iteration. Pass `storePath` to persist + continue the frontier across runs. Needs a *graded* metric, not binary. +- `dspy_bootstrap` — BootstrapFewShot: labeled + self-bootstrapped demos. Pass `dynamicPath` so the compiled module picks input-conditioned demos at run time. +- `dspy_replay_status` — inspect an AgentDB replay/frontier/dynamic-demo store (vector count, dimension, tiers, quantization). + +**Resources**: `dspy://optimizer-selection`, `dspy://experience-replay`, `dspy://traces/{runId}`. + +The handlers shell out to `npx ts-node` against `src/dspy/*` (your program + its `metric`) and to `dspy.ts`. Slash commands `/dspy-mipro`, `/dspy-gepa`, `/dspy-bootstrap` wrap the same flows for interactive use; the `optimizer-engineer` agent runs full campaigns. diff --git a/plugins/dspy-optimize/agents/optimizer-engineer.md b/plugins/dspy-optimize/agents/optimizer-engineer.md new file mode 100644 index 0000000..6413c85 --- /dev/null +++ b/plugins/dspy-optimize/agents/optimizer-engineer.md @@ -0,0 +1,14 @@ +--- +name: optimizer-engineer +description: Runs DSPy.ts optimizer campaigns — picks the optimizer (BootstrapFewShot / MIPROv2 / GEPA), sizes the budget, wires AgentDB experience replay / GEPA frontier persistence and a CompilationTracer, runs the compile, reads the trace, and iterates. Use to actually optimize a DSPy.ts program (not just design it). +--- +You run optimizer campaigns for DSPy.ts programs. + +PICK THE OPTIMIZER: +- **BootstrapFewShot** — cheapest. Few-shot demos (labeled + self-bootstrapped). Add `dynamicDemos: { store, k }` (an `AgentDBClient`) so the compiled module conditions its demos on the input. Start here. +- **MIPROv2** — proposes instructions *and* selects demos via a seeded random search over `numTrials`. Add `replayStore` (an `AgentDBClient`) + `replayTopK` so repeated compiles of the same task fingerprint warm-start from prior bests. Add a `CompilationTracer` (`{ store }`) — every trial is logged with a `causedBy` link; `tracer.causalChain(runId)` shows the path to the best. Good default for non-trivial tasks. +- **GEPA** — reflective Pareto evolution: each iteration picks a frontier parent, finds its weakest examples, asks the LM to mutate the instruction to fix them, evaluates, and updates a deduplicated frontier (`dominates` check). Add `frontierStore` to persist + continue across runs. Needs the *best* metric — its reflect step chases the lowest scorers, so a noisy metric chases noise. Use when you have a solid graded metric and want the prompt to genuinely evolve. + +BUDGET: BootstrapFewShot — minutes; MIPROv2 — `numTrials × |trainset|` metric calls (use a minibatch metric if the metric is an LM judge); GEPA — `numIterations × mutationsPerStep` LM calls + `numIterations × |trainset|` evals. + +LOOP: compile → read `opt.result` (best score, trials/reflections, `warmStarted`) → if flat, suspect the metric (binary? noisy?) before adding budget → re-run with the same replay/frontier store to build on it → `opt.save(...)`. Hand the saved JSON back; loadable with `opt.load(...)`. diff --git a/plugins/dspy-optimize/commands/dspy-bootstrap.md b/plugins/dspy-optimize/commands/dspy-bootstrap.md new file mode 100644 index 0000000..13ddb3a --- /dev/null +++ b/plugins/dspy-optimize/commands/dspy-bootstrap.md @@ -0,0 +1,12 @@ +--- +description: Compile a DSPy.ts program with BootstrapFewShot — labeled + self-bootstrapped demos — optionally with input-conditioned dynamic demo selection backed by an AgentDB vector store (the compiled module picks the demos nearest the current input at run time). +argument-hint: " [trainset: data.json] [--labeled N] [--bootstrapped N] [--dynamic path/to/agentdb] [--k N]" +--- +Optimize `src/dspy/.ts` with `BootstrapFewShot`. Parse `$ARGUMENTS` for the program, a trainset JSON, `--labeled` (max labeled demos), `--bootstrapped` (max self-bootstrapped demos), `--dynamic` (AgentDB path → `dynamicDemos.store`), `--k` (demos picked per input, default 1–2). + +1. Load the program + `metric` + the trainset. +2. `const store = dynamicPath ? new AgentDBClient({ storage: { path: dynamicPath } }) : undefined; await store?.init();` +3. `const opt = new BootstrapFewShot(metric, { maxLabeledDemos, maxBootstrappedDemos, dynamicDemos: store ? { store, k } : undefined });` +4. `const compiled = await opt.compile(program, trainset);` → a `BootstrapOptimizedModule`. +5. `opt.save('src/dspy/.bootstrap.json');` +6. If `--dynamic`: show that `await compiled.selectDemos(input)` returns the k-nearest demos for a given input, and `await compiled.run(input)` builds the prompt from just those (not a fixed set). Otherwise `selectDemos` returns the full fixed set. diff --git a/plugins/dspy-optimize/commands/dspy-gepa.md b/plugins/dspy-optimize/commands/dspy-gepa.md new file mode 100644 index 0000000..3d57808 --- /dev/null +++ b/plugins/dspy-optimize/commands/dspy-gepa.md @@ -0,0 +1,13 @@ +--- +description: Evolve a DSPy.ts program's prompt with GEPA — reflective mutation of a Pareto frontier, targeting the lowest-scoring examples each iteration; optionally persists the frontier to AgentDB so a later run continues the evolution. +argument-hint: " [trainset: data.json] [--iterations N] [--mutations N] [--frontier N] [--store path/to/agentdb]" +--- +Evolve `src/dspy/.ts` with `GEPA`. Parse `$ARGUMENTS` for the program, a trainset JSON, `--iterations` (default 12), `--mutations` (mutations per step, default 2), `--frontier` (frontier size, default 8), `--store` (AgentDB path → `frontierStore`). + +1. Load the program + `metric` (GEPA leans hard on a good, graded metric — see the `metric-design` skill in `dspy-core`) + the trainset. +2. `const store = storePath ? new AgentDBClient({ storage: { path: storePath } }) : undefined; await store?.init();` +3. `const opt = new GEPA(metric, { numIterations, mutationsPerStep, frontierSize, frontierStore: store, seed: 42 });` +4. `const compiled = await opt.compile(program, trainset);` +5. `opt.save('src/dspy/.gepa.json');` +6. Report `opt.result` — `{ best.instruction, best.meanScore, frontier.length, reflections (from → weakExamples → mutated per iteration), iterations, warmStarted }`. Show the frontier (instruction + meanScore) so the user can pick a different non-dominated candidate. +7. Re-run with the same `--store` → `warmStarted: true`, evolution continues from the persisted frontier. diff --git a/plugins/dspy-optimize/commands/dspy-mipro.md b/plugins/dspy-optimize/commands/dspy-mipro.md new file mode 100644 index 0000000..46ca488 --- /dev/null +++ b/plugins/dspy-optimize/commands/dspy-mipro.md @@ -0,0 +1,14 @@ +--- +description: Compile a DSPy.ts program with MIPROv2 — instruction proposal + demo bootstrapping + seeded search — optionally with an AgentDB experience-replay store so later compiles of the same task warm-start from the prior best. +argument-hint: " [trainset: data.json] [--trials N] [--candidates N] [--replay path/to/agentdb] [--trace]" +--- +Optimize `src/dspy/.ts` with `MIPROv2`. Parse `$ARGUMENTS` for the program, a trainset JSON (`[{input, output?}]`), `--trials` (default 12), `--candidates` (instruction candidates, default ~10), `--replay` (an AgentDB path → an `AgentDBClient` used as `replayStore`), and `--trace` (attach a `CompilationTracer`). + +1. Load the program + its `metric` + the trainset. +2. `const store = replayPath ? new AgentDBClient({ storage: { path: replayPath } }) : undefined; await store?.init();` + `const tracer = trace ? new CompilationTracer({ store }) : undefined;` +3. `const opt = new MIPROv2(metric, { numTrials, numCandidateInstructions, replayStore: store, replayTopK: 3, tracer, seed: 42 });` +4. `const compiled = await opt.compile(program, trainset);` +5. `opt.save('src/dspy/.optimized.json');` +6. Report `opt.result` — `{ instruction, demos.length, score, trials.length, warmStarted, recalledInstructions }`. If `--trace`, print the trial trace and `tracer.causalChain(runId)`. +7. Re-run later with the same `--replay` path → `warmStarted: true` and the search seeds from the prior best instruction. diff --git a/plugins/dspy-optimize/mcp/server.js b/plugins/dspy-optimize/mcp/server.js new file mode 100644 index 0000000..e17e0bf --- /dev/null +++ b/plugins/dspy-optimize/mcp/server.js @@ -0,0 +1,30 @@ +#!/usr/bin/env node +/** + * dspy-optimize MCP server — exposes DSPy.ts optimizer campaigns as MCP tools. + * Tools: dspy_mipro (MIPROv2 + experience replay), dspy_gepa (GEPA Pareto + * evolution), dspy_bootstrap (BootstrapFewShot + dynamic demos), dspy_replay_status + * (inspect an AgentDB replay/frontier store). + * Resources: dspy://optimizer-selection, dspy://experience-replay, dspy://traces/{runId}. + * + * Scaffold: the handlers shell out to `npx ts-node` against the project's + * src/dspy/* and to dspy.ts (MIPROv2 / GEPA / BootstrapFewShot, AgentDBClient, + * CompilationTracer). Flesh out the @modelcontextprotocol/sdk stdio wiring + handlers. + */ +'use strict'; +const TOOLS = [ + { name: 'dspy_mipro', description: 'Compile a DSPy.ts program with MIPROv2 (instruction proposal + demo bootstrapping + seeded search); optional AgentDB replayStore warm-starts repeated compiles of the same task. Returns {instruction, demos, score, trials, warmStarted, recalledInstructions}.', inputSchema: { type: 'object', properties: { program: { type: 'string' }, trainset: { type: 'string', description: 'path to [{input,output?}] JSON' }, numTrials: { type: 'number' }, numCandidateInstructions: { type: 'number' }, replayPath: { type: 'string', description: 'AgentDB path for experience replay' }, trace: { type: 'boolean' } }, required: ['program', 'trainset'] } }, + { name: 'dspy_gepa', description: 'Evolve a DSPy.ts program’s prompt with GEPA (reflective Pareto-frontier evolution targeting weak examples); optional AgentDB frontierStore persists + continues the evolution. Returns {best, frontier, reflections, iterations, warmStarted}.', inputSchema: { type: 'object', properties: { program: { type: 'string' }, trainset: { type: 'string' }, numIterations: { type: 'number' }, mutationsPerStep: { type: 'number' }, frontierSize: { type: 'number' }, storePath: { type: 'string', description: 'AgentDB path for the frontier' } }, required: ['program', 'trainset'] } }, + { name: 'dspy_bootstrap', description: 'Compile a DSPy.ts program with BootstrapFewShot (labeled + self-bootstrapped demos); optional AgentDB dynamicDemos store makes the compiled module pick input-conditioned demos at run time. Returns the saved optimized program path.', inputSchema: { type: 'object', properties: { program: { type: 'string' }, trainset: { type: 'string' }, maxLabeledDemos: { type: 'number' }, maxBootstrappedDemos: { type: 'number' }, dynamicPath: { type: 'string' }, k: { type: 'number' } }, required: ['program', 'trainset'] } }, + { name: 'dspy_replay_status', description: 'Inspect an AgentDB experience-replay / GEPA-frontier / dynamic-demo store: vector count, dimension, tier counts, quantization info.', inputSchema: { type: 'object', properties: { storePath: { type: 'string' } }, required: ['storePath'] } }, +]; +const RESOURCES = [ + { uri: 'dspy://optimizer-selection', name: 'Optimizer selection guide', description: 'BootstrapFewShot vs MIPROv2 vs GEPA — choosing and budgeting.', mimeType: 'text/markdown' }, + { uri: 'dspy://experience-replay', name: 'Experience replay guide', description: 'Wiring an AgentDBClient as replayStore / frontierStore / dynamicDemos.', mimeType: 'text/markdown' }, + { uri: 'dspy://traces/{runId}', name: 'Compilation trace', description: 'CompilationTracer run — trials with causedBy links; causalChain to the best.', mimeType: 'application/json' }, +]; +module.exports = { TOOLS, RESOURCES }; +// TODO: wire @modelcontextprotocol/sdk StdioServerTransport; handlers shell out to +// `npx ts-node` against src/dspy/* and dspy.ts (MIPROv2/GEPA/BootstrapFewShot, AgentDBClient, CompilationTracer). +if (require.main === module) { + process.stderr.write('[dspy-optimize mcp] scaffold — handlers shell out to `npx ts-node` against src/dspy/* and dspy.ts. Tools: ' + TOOLS.map(t => t.name).join(', ') + '\n'); +} diff --git a/plugins/dspy-optimize/skills/experience-replay/SKILL.md b/plugins/dspy-optimize/skills/experience-replay/SKILL.md new file mode 100644 index 0000000..2bb6a09 --- /dev/null +++ b/plugins/dspy-optimize/skills/experience-replay/SKILL.md @@ -0,0 +1,34 @@ +--- +name: experience-replay +version: "0.1.0" +author: rUv +tags: [dspy, agentdb, experience-replay, warm-start, miprov2, gepa] +description: > + How DSPy.ts optimizers use an AgentDBClient to remember prior compiles — MIPROv2 `replayStore` (warm-start from prior best instruction), GEPA `frontierStore` (persist + continue the Pareto frontier), BootstrapFewShot `dynamicDemos` (input-conditioned demos). + Use when: you re-compile a program over time, or want one program's optimization to benefit the next. +--- +# Experience replay with AgentDB + +DSPy.ts optimizers can take an `AgentDBClient` and persist their search so the *next* compile of the same task starts ahead. + +## Set up a store +```ts +import { AgentDBClient } from 'dspy.ts'; +const store = new AgentDBClient({ + vectorDimension: 64, // small is fine for fingerprints/instructions + storage: { path: '.dspy/replay' }, // or { inMemory: true } for tests + performance: { maxConcurrency: 1, cacheSize: 8, batchEnabled: true /*, quantization: 'rabitq', rerankFactor: 3 */ }, +}); +await store.init(); +``` + +## Wire it +- **MIPROv2** — `new MIPROv2(metric, { numTrials, replayStore: store, replayTopK: 3, tracer: new CompilationTracer({ store }) })`. The optimizer keys by a task fingerprint (signature + program name); a later compile recalls the top-`replayTopK` prior best instructions and seeds the search with them → `result.warmStarted === true`, `result.recalledInstructions >= 1`. Stores one `mipro-best` record per compile. +- **GEPA** — `new GEPA(metric, { numIterations, frontierStore: store })`. Persists `gepa-candidate` records; a later run rebuilds the frontier from them and keeps evolving → `result.warmStarted === true`. +- **BootstrapFewShot** — `new BootstrapFewShot(metric, { dynamicDemos: { store, k: 1 } })`. The compiled `BootstrapOptimizedModule` indexes the demos (`store.hashEmbed` → vectors) and, at run time, `selectDemos(input)` returns the k-nearest — the prompt is built from those, not a fixed set. + +## Notes +- A **different** program (different fingerprint) never pulls another task's history. +- Without a store, every compile is a cold start (`warmStarted: false`). +- For real semantic recall over text (not just hash fingerprints), an `AgentDBClient` initialized with an embedding service uses ONNX 384-dim embeddings; HNSW + RaBitQ + tiered storage (`working`/`short`/`long`, `searchTiered`, `promote`) keep it fast as the store grows. +- The `CompilationTracer` shares the same store — `compile-run` / `compile-trial` / `compile-run-end` records, with `causedBy` links you can walk via `tracer.causalChain(runId)`. diff --git a/plugins/dspy-optimize/skills/optimizer-selection/SKILL.md b/plugins/dspy-optimize/skills/optimizer-selection/SKILL.md new file mode 100644 index 0000000..d888e79 --- /dev/null +++ b/plugins/dspy-optimize/skills/optimizer-selection/SKILL.md @@ -0,0 +1,26 @@ +--- +name: optimizer-selection +version: "0.1.0" +author: rUv +tags: [dspy, optimizer, miprov2, gepa, bootstrap] +description: > + How to choose and budget a DSPy.ts optimizer — BootstrapFewShot vs MIPROv2 vs GEPA — and when to wire AgentDB replay / frontier persistence. + Use when: setting up `dspy-compile` / `dspy-mipro` / `dspy-gepa` / `dspy-bootstrap`, or when a compile won't improve. +--- +# Choosing a DSPy.ts optimizer + +| | `BootstrapFewShot` | `MIPROv2` | `GEPA` | +|---|---|---|---| +| What it tunes | demos (labeled + bootstrapped) | instruction **and** demos, via seeded search | instruction, via reflective Pareto evolution | +| Cost | minutes | `numTrials × |trainset|` metric calls | `numIterations × mutationsPerStep` LM calls + evals | +| AgentDB hook | `dynamicDemos: { store, k }` — input-conditioned demos at run time | `replayStore` + `replayTopK` — warm-start repeated compiles of the same task | `frontierStore` — persist + continue evolution | +| Best when | quick win, you have a few good examples | non-trivial task, you'll re-compile as data grows | you have a strong graded metric and want the prompt to evolve | +| Determinism | yes | yes (fixed `seed`) | yes (fixed `seed`) | + +## Rules of thumb +- **Start with BootstrapFewShot.** If it's enough, stop. Add `dynamicDemos` only if inputs are heterogeneous. +- **Reach for MIPROv2** when one instruction won't cover the task. Always attach a `CompilationTracer` so trials are inspectable; attach a `replayStore` if you'll re-compile. +- **Reach for GEPA** last — it's the most expensive and the most metric-sensitive, but it's the one that *rewrites* the prompt. Don't run it on a binary metric. +- **A flat `result.score` is almost always the metric, not the budget.** Binary metrics give no gradient; noisy metrics send GEPA chasing noise. Fix the metric (`dspy-core`'s `metric-design` skill) before adding trials. +- **Warm-start is per task fingerprint** — a different program (different signature/name) won't pull another task's bests. +- Always `opt.save(...)` the result; `opt.load(...)` restores the compiled program + the trace. diff --git a/plugins/dspy-rag/.claude-plugin/plugin.json b/plugins/dspy-rag/.claude-plugin/plugin.json new file mode 100644 index 0000000..03ae2a1 --- /dev/null +++ b/plugins/dspy-rag/.claude-plugin/plugin.json @@ -0,0 +1,10 @@ +{ + "name": "dspy-rag", + "description": "Retrieval-augmented generation for DSPy.ts — index a corpus into AgentDB (HNSW + RaBitQ + tiered storage), retrieve with MMR diversity (RetrieveModule), and wire it into a Retrieve → ChainOfThought Pipeline with grounded, cited answers. Includes the `rag-architect` agent and MCP tools for indexing/retrieving.", + "version": "0.1.0", + "author": { "name": "rUv", "url": "https://github.com/ruvnet" }, + "homepage": "https://github.com/ruvnet/dspy.ts", + "license": "MIT", + "keywords": ["dspy", "dspy.ts", "rag", "retrieval", "agentdb", "hnsw", "mmr", "rabitq", "chain-of-thought"], + "mcpServers": { "dspy-rag": { "command": "node", "args": ["${CLAUDE_PLUGIN_ROOT}/mcp/server.js"] } } +} diff --git a/plugins/dspy-rag/AGENTS.md b/plugins/dspy-rag/AGENTS.md new file mode 100644 index 0000000..f38879d --- /dev/null +++ b/plugins/dspy-rag/AGENTS.md @@ -0,0 +1,13 @@ +# dspy-rag — for Codex / MCP clients + +This plugin ships an MCP server (`mcp/server.js`, command `node ${CLAUDE_PLUGIN_ROOT}/mcp/server.js`) for DSPy.ts retrieval-augmented generation over AgentDB. + +**Tools** +- `dspy_index` — chunk documents (size/overlap), embed (ONNX 384-dim, hashEmbed fallback), store in an AgentDB corpus with HNSW indexing, optional RaBitQ quantization, and a tier (`working`/`short`/`long`). +- `dspy_retrieve` — query a corpus the way `RetrieveModule` would: top-k passages + scores, MMR diversity rerank over an over-fetched candidate set, and the assembled context string. Use this to debug retrieval before tuning the answerer. +- `dspy_rag_run` — run a DSPy.ts RAG program (`Pipeline` of `RetrieveModule → ChainOfThought`) on a question; returns `{answer, citations, passages, context}`. +- `dspy_corpus_status` — corpus stats (vectors, dimension, tier counts, quantization). + +**Resources**: `dspy://chunking-strategy`, `dspy://grounding-and-citations`, `dspy://corpus/{path}/stats`. + +Handlers shell out to `npx ts-node` against `src/dspy/*` and `dspy.ts`. Slash commands `/dspy-index`, `/dspy-rag-new`, `/dspy-retrieve` wrap the same flows interactively; the `rag-architect` agent designs+builds a full pipeline (corpus → RetrieveModule → ChainOfThought → grounding metric), then hand off to `dspy-optimize`'s `/dspy-mipro`/`/dspy-gepa` to tune the answerer. diff --git a/plugins/dspy-rag/agents/rag-architect.md b/plugins/dspy-rag/agents/rag-architect.md new file mode 100644 index 0000000..65a7d3f --- /dev/null +++ b/plugins/dspy-rag/agents/rag-architect.md @@ -0,0 +1,14 @@ +--- +name: rag-architect +description: Designs and builds DSPy.ts RAG pipelines — chooses chunking, builds the AgentDB corpus (HNSW / RaBitQ / tiers), configures RetrieveModule (k, MMR, over-fetch), wires Retrieve → ChainOfThought in a Pipeline, and writes a grounding metric. Use to turn "answer questions over these docs" into a working DSPy.ts program. +--- +You build RAG pipelines on DSPy.ts + AgentDB. + +DESIGN: +1. **Corpus** — an `AgentDBClient` (`vectorDimension: 384` ⇒ ONNX embeddings; `hashEmbed` fallback). `storage.path` for a durable corpus; `performance.quantization: 'rabitq'` (+ `rerankFactor: 3`) for big corpora — `coarseThenRerank` at query time. Tiers: index into `long`; `promote` hot chunks to `working`; `evictTier` stale ones. +2. **Chunking** — size ≈ a coherent unit (paragraph/section), `overlap` ≈ 10–20% so a span isn't split mid-thought. Keep `source` + `offset` in metadata so the answerer can cite. (See the `chunking-strategy` skill.) +3. **Retrieve** — `new RetrieveModule({ client, k, useMMR: true, mmrLambda, overFetchFactor, textField })`. `k` = how much context the answerer can use well (4–8). `mmrLambda` near 0.5: lower → more diverse (good when chunks repeat), higher → more on-topic. `overFetchFactor` ≥ 3 so MMR has candidates to choose from. `run({ query })` → `{ passages, context }`. +4. **Answerer** — `ChainOfThought`, signature `{ question, context } → { answer, citations }`. The prompt MUST say "use only the context" and "cite the source of each claim". Compose `Pipeline([retrieveStep, cotStep])` (retrieveStep maps `{question}→{context}`). +5. **Metric** — score answer quality **and** faithfulness/citation. A correct-but-uncited answer should not max out; an answer that cites context it didn't use should be penalised. (See the `grounding-and-citations` skill.) + +DELIVER: the corpus build (`/dspy-index`), the program file, the metric, a small eval set. Then tune the answerer with `/dspy-mipro` or `/dspy-gepa` (the retriever is config, not learned — tune `k`/`mmrLambda` by hand against `/dspy-retrieve`). diff --git a/plugins/dspy-rag/commands/dspy-index.md b/plugins/dspy-rag/commands/dspy-index.md new file mode 100644 index 0000000..2bbd76d --- /dev/null +++ b/plugins/dspy-rag/commands/dspy-index.md @@ -0,0 +1,10 @@ +--- +description: Build (or extend) an AgentDB corpus for RAG — chunk documents, embed them, and store them with HNSW indexing, optional RaBitQ quantization, and tiered storage. +argument-hint: " [--chunk-size N] [--overlap N] [--tier working|short|long] [--rabitq]" +--- +Index documents into an `AgentDBClient` at ``. Parse `$ARGUMENTS` for the corpus path, the source (a directory, a glob, or a file), `--chunk-size` (default ~800 chars), `--overlap` (default ~120), `--tier` (default `long` — the durable tier), `--rabitq` (1-bit quantization, ~32× smaller, `coarseThenRerank` at query time). + +1. `const store = new AgentDBClient({ vectorDimension: 384, storage: { path: corpusPath }, performance: { batchEnabled: true${RABITQ:+, quantization: 'rabitq', rerankFactor: 3} } }); await store.init();` (384-dim ⇒ uses the ONNX embedding service; falls back to `hashEmbed` if unavailable). +2. Read each document; chunk it (size/overlap — see the `chunking-strategy` skill); for each chunk: `await store.storeText(chunkText, { source: file, offset, ... }, { tier })` (or `store.batchStore([...])`). +3. Report: documents read, chunks stored, `store.getStats()` (`totalVectors`, dimension), `store.tierCounts()`, and `store.quantizationInfo()` if `--rabitq`. +4. Re-run on the same path to append more sources. Use `store.promote(...)` / `store.evictTier(...)` to manage tiers over time. diff --git a/plugins/dspy-rag/commands/dspy-rag-new.md b/plugins/dspy-rag/commands/dspy-rag-new.md new file mode 100644 index 0000000..3b14568 --- /dev/null +++ b/plugins/dspy-rag/commands/dspy-rag-new.md @@ -0,0 +1,13 @@ +--- +description: Scaffold a DSPy.ts RAG program — a RetrieveModule (MMR) over an AgentDB corpus feeding a ChainOfThought answerer, composed in a Pipeline, plus a grounding metric. +argument-hint: " [--k N] [--mmr-lambda 0..1] [--overfetch N]" +--- +Scaffold `src/dspy/.ts` — a `Pipeline` of `RetrieveModule → ChainOfThought`. Parse `$ARGUMENTS` for the name, the corpus path, `--k` (passages, default 4), `--mmr-lambda` (relevance↔diversity, default 0.5), `--overfetch` (over-fetch factor before MMR rerank, default 3). + +1. Imports from `'dspy.ts'`: `RetrieveModule`, `ChainOfThought`, `Pipeline`, `AgentDBClient`, `configureLM`, types `Signature`, `MetricFunction`. +2. `const corpus = new AgentDBClient({ vectorDimension: 384, storage: { path: corpusPath } }); await corpus.init();` +3. `const retrieve = new RetrieveModule({ client: corpus, k, useMMR: true, mmrLambda, overFetchFactor: overfetch, textField: 'text' });` — `run({ query })` → `{ passages, context }`. +4. ChainOfThought signature: inputs `{ question, context }`, outputs `{ answer, citations }`; prompt uses `context` and instructs "answer only from context; cite the source of each claim". +5. `const program = new Pipeline([ /* retrieve step (maps {question}→{context}) */, /* cot step */ ]);` Export `program` + a `metric(input, output, gold?)` that scores answer quality **and** citation faithfulness (see the `grounding-and-citations` skill — never a vague-but-uncited answer). +6. Create `src/dspy/.spec.ts` — `configureLM(new DummyLM())`, index a tiny corpus, run the program, assert `passages.length <= k` and the output shape. +7. Print next steps: `/dspy-index ...` to fill the corpus, then `/dspy-compile mipro` (or `/dspy-gepa`) to tune the answerer. diff --git a/plugins/dspy-rag/commands/dspy-retrieve.md b/plugins/dspy-rag/commands/dspy-retrieve.md new file mode 100644 index 0000000..068c49a --- /dev/null +++ b/plugins/dspy-rag/commands/dspy-retrieve.md @@ -0,0 +1,11 @@ +--- +description: Query an AgentDB RAG corpus and show what RetrieveModule would feed the answerer — the top-k passages, their scores, and how MMR re-ranked them for diversity. +argument-hint: " \"\" [--k N] [--mmr-lambda 0..1] [--no-mmr]" +--- +Run a retrieval against the corpus at ``. Parse `$ARGUMENTS` for the corpus path, the query string, `--k` (default 4), `--mmr-lambda` (default 0.5), `--no-mmr` (plain top-k, no diversity rerank). + +1. `const corpus = new AgentDBClient({ vectorDimension: 384, storage: { path: corpusPath } }); await corpus.init();` +2. `const retrieve = new RetrieveModule({ client: corpus, k, useMMR: !noMmr, mmrLambda, overFetchFactor: 3, textField: 'text' });` +3. `const { passages, context } = await retrieve.run({ query });` +4. Print each passage: rank, score, source/metadata, a snippet. Then show the assembled `context` string (what ChainOfThought sees). If MMR is on, note which over-fetched candidates were dropped for redundancy. +5. This is a debugging view — if the right passages aren't in the top-k, fix the corpus (chunking, more sources) or `--k`/`--overfetch` before tuning the answerer. diff --git a/plugins/dspy-rag/mcp/server.js b/plugins/dspy-rag/mcp/server.js new file mode 100644 index 0000000..2674b8c --- /dev/null +++ b/plugins/dspy-rag/mcp/server.js @@ -0,0 +1,30 @@ +#!/usr/bin/env node +/** + * dspy-rag MCP server — exposes DSPy.ts retrieval-augmented generation as MCP tools. + * Tools: dspy_index (chunk + embed + store a corpus in AgentDB), dspy_retrieve + * (top-k + MMR for a query), dspy_rag_run (run a Retrieve→ChainOfThought program), + * dspy_corpus_status (corpus stats: vectors, dimension, tiers, quantization). + * Resources: dspy://chunking-strategy, dspy://grounding-and-citations, dspy://corpus/{path}/stats. + * + * Scaffold: handlers shell out to `npx ts-node` against src/dspy/* and dspy.ts + * (RetrieveModule, ChainOfThought, Pipeline, AgentDBClient). Flesh out the + * @modelcontextprotocol/sdk stdio wiring + handlers. + */ +'use strict'; +const TOOLS = [ + { name: 'dspy_index', description: 'Build/extend an AgentDB RAG corpus: chunk documents (size/overlap), embed (ONNX 384-dim, hashEmbed fallback), store with HNSW + optional RaBitQ + tier. Returns {documents, chunks, totalVectors, tierCounts}.', inputSchema: { type: 'object', properties: { corpusPath: { type: 'string' }, source: { type: 'string', description: 'dir | glob | file' }, chunkSize: { type: 'number' }, overlap: { type: 'number' }, tier: { type: 'string', enum: ['working', 'short', 'long'] }, rabitq: { type: 'boolean' } }, required: ['corpusPath', 'source'] } }, + { name: 'dspy_retrieve', description: 'Query an AgentDB RAG corpus the way RetrieveModule would: top-k passages with scores, MMR diversity rerank over an over-fetched candidate set, and the assembled context string.', inputSchema: { type: 'object', properties: { corpusPath: { type: 'string' }, query: { type: 'string' }, k: { type: 'number' }, mmrLambda: { type: 'number' }, useMMR: { type: 'boolean' }, overFetchFactor: { type: 'number' } }, required: ['corpusPath', 'query'] } }, + { name: 'dspy_rag_run', description: 'Run a DSPy.ts RAG program (a Pipeline of RetrieveModule → ChainOfThought) on a question; returns {answer, citations, passages, context}.', inputSchema: { type: 'object', properties: { program: { type: 'string', description: 'src/dspy/.ts' }, question: { type: 'string' } }, required: ['program', 'question'] } }, + { name: 'dspy_corpus_status', description: 'AgentDB corpus stats: totalVectors, vectorDimension, tier counts, RaBitQ quantization info.', inputSchema: { type: 'object', properties: { corpusPath: { type: 'string' } }, required: ['corpusPath'] } }, +]; +const RESOURCES = [ + { uri: 'dspy://chunking-strategy', name: 'Chunking strategy guide', description: 'Chunk size/overlap/metadata/tier choices for a DSPy.ts + AgentDB RAG corpus.', mimeType: 'text/markdown' }, + { uri: 'dspy://grounding-and-citations', name: 'Grounding & citations guide', description: 'Answerer signature/prompt/metric for grounded, cited RAG answers.', mimeType: 'text/markdown' }, + { uri: 'dspy://corpus/{path}/stats', name: 'Corpus stats', description: 'Live AgentDB corpus stats for a given path.', mimeType: 'application/json' }, +]; +module.exports = { TOOLS, RESOURCES }; +// TODO: wire @modelcontextprotocol/sdk StdioServerTransport; handlers shell out to +// `npx ts-node` against src/dspy/* and dspy.ts (RetrieveModule, ChainOfThought, Pipeline, AgentDBClient). +if (require.main === module) { + process.stderr.write('[dspy-rag mcp] scaffold — handlers shell out to `npx ts-node` against src/dspy/* and dspy.ts. Tools: ' + TOOLS.map(t => t.name).join(', ') + '\n'); +} diff --git a/plugins/dspy-rag/skills/chunking-strategy/SKILL.md b/plugins/dspy-rag/skills/chunking-strategy/SKILL.md new file mode 100644 index 0000000..122a76d --- /dev/null +++ b/plugins/dspy-rag/skills/chunking-strategy/SKILL.md @@ -0,0 +1,27 @@ +--- +name: chunking-strategy +version: "0.1.0" +author: rUv +tags: [dspy, rag, chunking, agentdb, retrieval] +description: > + How to chunk documents for a DSPy.ts + AgentDB RAG corpus — size, overlap, metadata, and tier choice — so RetrieveModule + MMR returns useful, citable context. + Use when: running `/dspy-index`, or when retrieval keeps missing the right passage. +--- +# Chunking for RAG + +You're filling an `AgentDBClient` corpus that a `RetrieveModule` will query. The chunk is the unit of retrieval *and* the unit of citation. + +## Rules of thumb +- **Chunk on natural boundaries.** A section, a paragraph, a list item — not a fixed byte window that slices a sentence in half. Fixed-size is the fallback, not the goal. +- **Size ≈ one coherent thought**, ~500–1000 chars for prose; smaller for dense reference text, larger for narrative. Too small ⇒ no context; too large ⇒ the answerer drowns and `k` passages blow the prompt budget. +- **Overlap ~10–20%** so a fact spanning a boundary appears whole in at least one chunk. `dspy.ts`'s embeddings package supports configurable overlap. +- **Keep metadata: `{ source, offset, title?, section? }`.** The answerer's signature has a `citations` output — it can only cite what you stored. `storeText(text, metadata, { tier })`. +- **Tier on durability/heat.** Index into `long`. `promote(...)` chunks that get hit a lot to `working`. `evictTier('short', { maxAgeMs })` for transient stuff. `searchTiered` queries across tiers. +- **Dedup before indexing.** Near-duplicate chunks waste the `k` budget; MMR helps at query time, but it's cheaper to not store them. +- **Big corpus ⇒ `quantization: 'rabitq'`** (~32× smaller, 1-bit). `coarseThenRerank` over-fetches on the packed bits then re-ranks the shortlist on full vectors — set `rerankFactor` ≥ 3. + +## Sizing `RetrieveModule` +- `k`: 4–8. More isn't better — past ~8 the answerer ignores the tail. +- `overFetchFactor`: ≥ 3, so MMR picks `k` diverse passages from `k × factor` candidates. +- `mmrLambda`: ~0.5. Drop toward 0.3 when chunks are repetitive; raise toward 0.7 when on-topic-ness matters more than coverage. +- Validate with `/dspy-retrieve ""` *before* tuning the answerer — a retriever that doesn't surface the answer can't be fixed downstream. diff --git a/plugins/dspy-rag/skills/grounding-and-citations/SKILL.md b/plugins/dspy-rag/skills/grounding-and-citations/SKILL.md new file mode 100644 index 0000000..42daf18 --- /dev/null +++ b/plugins/dspy-rag/skills/grounding-and-citations/SKILL.md @@ -0,0 +1,40 @@ +--- +name: grounding-and-citations +version: "0.1.0" +author: rUv +tags: [dspy, rag, grounding, citations, metric, faithfulness] +description: > + How to design the answerer signature, prompt, and metric for a DSPy.ts RAG pipeline so answers are grounded in the retrieved context and cite their sources — and so an optimizer can't game it. + Use when: writing the ChainOfThought step / metric for a RAG program, or when answers drift off-context. +--- +# Grounding & citations + +The retriever gives you `{ passages, context }`. The answerer's job is to answer **from that context** and say **where**. + +## Signature & prompt +- Signature: inputs `{ question, context }`, outputs `{ answer, citations }` (`citations: object` — e.g. an array of `{ source, span }`). +- The prompt must say, explicitly: + - "Answer **only** using the context below. If the context doesn't contain the answer, say so." + - "For each claim in the answer, cite the source it came from." +- Use `ChainOfThought` (not `Predict`) — the reasoning step is where it decides which passages support the answer; that reasoning is visible in the output and useful for the metric. + +## Metric — score quality AND faithfulness +A naive `answer === gold ? 1 : 0` metric lets the optimizer produce confident, uncited, sometimes-wrong answers. Score both axes, in [0,1]: +```ts +const metric: MetricFunction = (input, out, gold?) => { + if (!out?.answer) return 0; + let q = gold ? scoreAnswer(out.answer, gold.answer) : heuristicAnswerScore(out.answer); // 0..1 + // faithfulness: are the cited sources actually in the retrieved context, and does the answer stick to them? + const cited = (out.citations ?? []) as { source: string }[]; + const inContext = cited.length > 0 && cited.every(c => input.context.includes(c.source) /* or check passage ids */); + const f = cited.length === 0 ? 0.4 // answered but didn't cite → capped + : inContext ? 1.0 // cited real context + : 0.2; // cited something not retrieved → hallucinated cite + return 0.6 * q + 0.4 * f; // never let a great-but-uncited answer hit 1.0 +}; +``` +## Rules of thumb +- **Reward partial correctness** on the quality axis (overlap/F1, not exact-match) — optimizers need a gradient. (See `dspy-core`'s `metric-design`.) +- **Penalise unsupported citations harder than missing ones** — a fake cite is worse than no cite. +- **"I don't know" when the context lacks the answer is the *correct* output** — your metric and prompt should both allow it; don't train the model to bluff. +- Tune the **retriever** (`k`, `mmrLambda`, chunking) by hand against `/dspy-retrieve`; tune the **answerer's instruction** with `/dspy-mipro` or `/dspy-gepa` against this metric. diff --git a/plugins/dspy-react/.claude-plugin/plugin.json b/plugins/dspy-react/.claude-plugin/plugin.json new file mode 100644 index 0000000..cdb0d31 --- /dev/null +++ b/plugins/dspy-react/.claude-plugin/plugin.json @@ -0,0 +1,10 @@ +{ + "name": "dspy-react", + "description": "ReAct agents for DSPy.ts — reasoning + acting loops over a tool registry, with ReActReflexion: recall prior lessons before acting, record episodes after, and promote recurring successful action sequences into reusable skills (backed by AgentDB). Includes the `react-engineer` agent and MCP tools for running/inspecting ReAct programs.", + "version": "0.1.0", + "author": { "name": "rUv", "url": "https://github.com/ruvnet" }, + "homepage": "https://github.com/ruvnet/dspy.ts", + "license": "MIT", + "keywords": ["dspy", "dspy.ts", "react", "agent", "tool-use", "reflexion", "agentdb", "skill-library"], + "mcpServers": { "dspy-react": { "command": "node", "args": ["${CLAUDE_PLUGIN_ROOT}/mcp/server.js"] } } +} diff --git a/plugins/dspy-react/AGENTS.md b/plugins/dspy-react/AGENTS.md new file mode 100644 index 0000000..1352c29 --- /dev/null +++ b/plugins/dspy-react/AGENTS.md @@ -0,0 +1,13 @@ +# dspy-react — for Codex / MCP clients + +This plugin ships an MCP server (`mcp/server.js`, command `node ${CLAUDE_PLUGIN_ROOT}/mcp/server.js`) for DSPy.ts ReAct agents with reflexion. + +**Tools** +- `dspy_react_new` — scaffold a ReAct agent: a `Signature`, a tool registry (`{name, description, handler}` each), the `ReAct` module, and optional `ReActReflexion` wired to an AgentDB store. +- `dspy_react_run` — run a ReAct program on a task; returns the full trace: recalled lessons, every `{thought, action:{tool,args}, observation}` step, the final `{answer, reasoning, steps}`, and what was recorded (episode stored, skill promoted). +- `dspy_reflexion_recall` — for a `taskKey`, the lessons (from failed episodes) and skills (promoted successful sequences) that would be injected before a run. +- `dspy_reflexion_status` — reflexion store stats (lessons / skills / episodes / tiers / quantization). + +**Resources**: `dspy://tool-design`, `dspy://reflexion-loop`, `dspy://reflexion/{path}/lessons`. + +Handlers shell out to `npx ts-node` against `src/dspy/*` and `dspy.ts`. Slash commands `/dspy-react-new`, `/dspy-react-run`, `/dspy-reflexion` wrap the same flows interactively; the `react-engineer` agent designs the tool registry, sets `maxSteps`, wires reflexion, and iterates on traces. Tune the thought prompt with `dspy-optimize`'s `/dspy-mipro`/`/dspy-gepa`. diff --git a/plugins/dspy-react/agents/react-engineer.md b/plugins/dspy-react/agents/react-engineer.md new file mode 100644 index 0000000..c170d92 --- /dev/null +++ b/plugins/dspy-react/agents/react-engineer.md @@ -0,0 +1,14 @@ +--- +name: react-engineer +description: Builds and tunes DSPy.ts ReAct agents — designs the tool registry, sets the step budget, wires ReActReflexion to AgentDB so the agent learns lessons and promotes skills, runs traces, and iterates on tool descriptions and the thought prompt. Use to turn "an agent that uses these tools to do X" into a working, self-improving DSPy.ts program. +--- +You build ReAct agents on DSPy.ts. + +DESIGN: +1. **Signature** — usually `{ question } → { answer }`; `ReAct` also exposes `reasoning: string` and `steps: object[]` automatically. Add task-specific outputs only if you'll score them. +2. **Tools** — each `{ name, description, handler }`. The **description is the entire interface the model sees** — say *what it does, what args it takes, and when to use it vs the others*. Keep the set small (3–6); overlapping tools cause thrash. Handlers must be robust — a thrown error becomes an observation the agent has to recover from. (See the `tool-design` skill.) +3. **Step budget** — `maxSteps` 4–8. Too low ⇒ it can't finish; too high ⇒ it wanders. If it routinely hits the cap, the tools or the prompt are wrong, not the cap. +4. **Reflexion** — `new ReActReflexion({ store: agentDbClient, recallK, skillThreshold })`. On `run()`: `recall(taskKey)` surfaces lessons + matched skills into the thought prompt; after, `recordEpisode(taskKey, { success, steps, critique })` stores a `react-reflexion` record on failure (with the critique) and, when a successful step sequence has been seen ≥ `skillThreshold` times, promotes a `react-skill`. The store is keyed by `taskKey` — lessons stay scoped to their task. Use a persistent `storage.path` so learning survives restarts; HNSW + RaBitQ keep recall fast as it grows. +5. **Tuning** — the *retriever-like* parts (tools, `maxSteps`, `recallK`) you tune by hand against `/dspy-react-run` traces. The *thought prompt* you can tune with `/dspy-mipro` / `/dspy-gepa` against a metric over a task set. + +LOOP: scaffold → implement handlers → `/dspy-react-run` on real tasks → read the trace (which tool got picked wrong? did it recover from an error observation?) → fix tool descriptions → watch lessons/skills accrue via `/dspy-reflexion` → tune the thought prompt last. Deliver the program file, the tool registry, and the reflexion store path. diff --git a/plugins/dspy-react/commands/dspy-react-new.md b/plugins/dspy-react/commands/dspy-react-new.md new file mode 100644 index 0000000..ddf554a --- /dev/null +++ b/plugins/dspy-react/commands/dspy-react-new.md @@ -0,0 +1,13 @@ +--- +description: Scaffold a DSPy.ts ReAct agent — a Signature, a tool registry (each tool a typed name/description/handler), a ReAct module, and optionally ReActReflexion wired to an AgentDB store so it learns from past episodes. +argument-hint: " [signature: question -> answer] [--tools tool1,tool2,...] [--reflexion path/to/agentdb] [--max-steps N]" +--- +Scaffold `src/dspy/.ts` — a `ReAct` agent. Parse `$ARGUMENTS` for the name, an optional `in -> out` signature (default `question -> answer`), `--tools` (a comma list of tool names to stub), `--reflexion` (an AgentDB path → a `ReActReflexion` store), `--max-steps` (default 6). + +1. Imports from `'dspy.ts'`: `ReAct`, `ReActReflexion`, `AgentDBClient`, `configureLM`, types `Signature`, `MetricFunction`. +2. Define the tool registry — for each `--tools` entry: `{ name, description: 'what it does + when to use it', handler: async (args) => /* ... */ }`. (See the `tool-design` skill — descriptions are the only thing the model sees.) +3. `const store = reflexionPath ? new AgentDBClient({ vectorDimension: 384, storage: { path: reflexionPath } }) : undefined; await store?.init();` + `const reflexion = store ? new ReActReflexion({ store, recallK: 3, skillThreshold: 3 }) : undefined;` +4. `const program = new ReAct({ name, signature, tools, maxSteps, reflexion });` — `run(input)` recalls prior lessons (into the thought prompt), loops thought→action→observation, then records the episode (`recordEpisode(taskKey, { success, steps, critique })`); a recurring successful step sequence (seen ≥ `skillThreshold`) is promoted to a `react-skill` record. +5. Create `src/dspy/.spec.ts` — `configureLM(new DummyLM())`, register a trivial tool, run, assert the output shape and that `steps` is an array. +6. Print next steps: implement the tool handlers, then `/dspy-react-run ""`; tune the thought prompt with `/dspy-mipro` (the reflexion store grows on its own as you run). diff --git a/plugins/dspy-react/commands/dspy-react-run.md b/plugins/dspy-react/commands/dspy-react-run.md new file mode 100644 index 0000000..f31dace --- /dev/null +++ b/plugins/dspy-react/commands/dspy-react-run.md @@ -0,0 +1,12 @@ +--- +description: Run a DSPy.ts ReAct program on a task and show the full trace — recalled lessons, each thought/action/observation step, the final answer, and what (if anything) was recorded to the reflexion store. +argument-hint: " \"\" [--no-reflexion] [--max-steps N]" +--- +Run `src/dspy/.ts`. Parse `$ARGUMENTS` for the program, the task string, `--no-reflexion` (skip recall + recording for this run), `--max-steps` (override). + +1. Load the program (and its `ReActReflexion` store, unless `--no-reflexion`). +2. Before the loop: print the lessons `reflexion.recall(taskKey)` surfaced (and any matched skills) — these go into the thought prompt. +3. `const out = await program.run({ /* the task */ });` +4. Print each step: `{ thought, action: { tool, args }, observation }`. Then the final `{ answer, reasoning, steps }`. +5. After the loop: report what `recordEpisode` did — episode stored, and whether a `react-skill` was promoted (a successful step sequence now seen ≥ `skillThreshold` times). Fallback answers count as `reachedAnswer: false`. +6. If it loops without converging: the tool descriptions are usually the problem (the model can't tell which tool to use) — see the `tool-design` skill. diff --git a/plugins/dspy-react/commands/dspy-reflexion.md b/plugins/dspy-react/commands/dspy-reflexion.md new file mode 100644 index 0000000..c8b615b --- /dev/null +++ b/plugins/dspy-react/commands/dspy-reflexion.md @@ -0,0 +1,12 @@ +--- +description: Inspect or manage a DSPy.ts ReActReflexion store — list the lessons learned from failed episodes, the skills promoted from recurring successful sequences, and the raw episode history. +argument-hint: " [--lessons] [--skills] [--episodes] [--task ]" +--- +Open the `AgentDBClient` at `` and report what `ReActReflexion` has accumulated. Parse `$ARGUMENTS` for the store path, optional `--task` filter, and which views to show (default: all). + +1. `const store = new AgentDBClient({ vectorDimension: 384, storage: { path } }); await store.init(); const reflexion = new ReActReflexion({ store });` +2. `--lessons` — `reflexion.lessonsText(taskKey?)` / `recall(taskKey)`: critiques distilled from failed episodes (the `react-reflexion` records). These are injected before future runs of the same task. +3. `--skills` — `reflexion.getSkills(taskKey?)`: action sequences promoted because they succeeded ≥ `skillThreshold` times (the `react-skill` records). +4. `--episodes` — the raw episode history (vector-searchable via the store). +5. `store.getStats()` / `store.tierCounts()` for size. Note: the store keys by `taskKey`, so lessons from one task don't bleed into another. +6. This is the "what has the agent learned" view — pair it with `/dspy-react-run` to watch new lessons/skills appear. diff --git a/plugins/dspy-react/mcp/server.js b/plugins/dspy-react/mcp/server.js new file mode 100644 index 0000000..818b3a9 --- /dev/null +++ b/plugins/dspy-react/mcp/server.js @@ -0,0 +1,31 @@ +#!/usr/bin/env node +/** + * dspy-react MCP server — exposes DSPy.ts ReAct agents (+ reflexion) as MCP tools. + * Tools: dspy_react_new (scaffold a ReAct agent + tool registry [+ reflexion]), + * dspy_react_run (run a ReAct program on a task, return the full trace), + * dspy_reflexion_recall (lessons + skills for a taskKey), dspy_reflexion_status + * (reflexion store: lessons/skills/episode counts, tiers). + * Resources: dspy://tool-design, dspy://reflexion-loop, dspy://reflexion/{path}/lessons. + * + * Scaffold: handlers shell out to `npx ts-node` against src/dspy/* and dspy.ts + * (ReAct, ReActReflexion, AgentDBClient). Flesh out the @modelcontextprotocol/sdk + * stdio wiring + handlers. + */ +'use strict'; +const TOOLS = [ + { name: 'dspy_react_new', description: 'Scaffold a DSPy.ts ReAct agent: a Signature, a tool registry (each tool {name, description, handler}), the ReAct module, and optional ReActReflexion wired to an AgentDB store. Returns the generated file paths.', inputSchema: { type: 'object', properties: { name: { type: 'string' }, signature: { type: 'string', description: '"in1,in2 -> out1" (default "question -> answer")' }, tools: { type: 'array', items: { type: 'string' }, description: 'tool names to stub' }, reflexionPath: { type: 'string', description: 'AgentDB path for the reflexion store' }, maxSteps: { type: 'number' } }, required: ['name'] } }, + { name: 'dspy_react_run', description: 'Run a DSPy.ts ReAct program on a task; returns {answer, reasoning, steps:[{thought,action,observation}], recalledLessons, recordedEpisode, promotedSkill}.', inputSchema: { type: 'object', properties: { program: { type: 'string', description: 'src/dspy/.ts' }, task: { type: 'string' }, useReflexion: { type: 'boolean' }, maxSteps: { type: 'number' } }, required: ['program', 'task'] } }, + { name: 'dspy_reflexion_recall', description: 'For a taskKey, return what ReActReflexion would inject before a run: distilled lessons (from failed episodes) and matched skills (promoted successful sequences).', inputSchema: { type: 'object', properties: { storePath: { type: 'string' }, taskKey: { type: 'string' }, recallK: { type: 'number' } }, required: ['storePath', 'taskKey'] } }, + { name: 'dspy_reflexion_status', description: 'ReActReflexion store stats: lesson count, promoted-skill count, episode count, tier counts, quantization info.', inputSchema: { type: 'object', properties: { storePath: { type: 'string' } }, required: ['storePath'] } }, +]; +const RESOURCES = [ + { uri: 'dspy://tool-design', name: 'Tool design guide', description: 'Designing the tool registry for a DSPy.ts ReAct agent — names, descriptions, args, error handling.', mimeType: 'text/markdown' }, + { uri: 'dspy://reflexion-loop', name: 'Reflexion loop guide', description: 'How ReActReflexion recalls lessons, records episodes, and promotes skills.', mimeType: 'text/markdown' }, + { uri: 'dspy://reflexion/{path}/lessons', name: 'Reflexion lessons', description: 'Live lessons + promoted skills from a reflexion store.', mimeType: 'application/json' }, +]; +module.exports = { TOOLS, RESOURCES }; +// TODO: wire @modelcontextprotocol/sdk StdioServerTransport; handlers shell out to +// `npx ts-node` against src/dspy/* and dspy.ts (ReAct, ReActReflexion, AgentDBClient). +if (require.main === module) { + process.stderr.write('[dspy-react mcp] scaffold — handlers shell out to `npx ts-node` against src/dspy/* and dspy.ts. Tools: ' + TOOLS.map(t => t.name).join(', ') + '\n'); +} diff --git a/plugins/dspy-react/skills/reflexion-loop/SKILL.md b/plugins/dspy-react/skills/reflexion-loop/SKILL.md new file mode 100644 index 0000000..e6a375f --- /dev/null +++ b/plugins/dspy-react/skills/reflexion-loop/SKILL.md @@ -0,0 +1,31 @@ +--- +name: reflexion-loop +version: "0.1.0" +author: rUv +tags: [dspy, react, reflexion, agentdb, skill-library, learning] +description: > + How DSPy.ts ReActReflexion makes a ReAct agent learn — recall lessons before acting, record episodes after, promote recurring successful action sequences into skills — all backed by AgentDB. + Use when: wiring reflexion into a ReAct agent, or deciding `recallK` / `skillThreshold` / the store layout. +--- +# The reflexion loop + +`ReActReflexion({ store, recallK, skillThreshold })` wraps a `ReAct` agent with memory. The store is an `AgentDBClient` (use a persistent `storage.path`). + +## What happens on `run(input)` +1. **Recall** — `recall(taskKey)` vector-searches the store for prior `react-reflexion` (lessons) and `react-skill` (promoted sequences) records for this `taskKey`, returns `{ lessons, skills }`. `buildThoughtPrompt` injects `priorLessons` so the agent starts with hindsight. +2. **Act** — the normal thought → action → observation loop, up to `maxSteps`. +3. **Record** — `recordEpisode(taskKey, { success, steps, critique })`: + - on **failure** (`reachedAnswer === false`, i.e. the answer came from the fallback) with a critique → a `react-reflexion` record (the lesson) is stored. + - on **success** → the step sequence is counted; once a given sequence has been seen ≥ `skillThreshold` times, it's promoted to a `react-skill` record (a reusable plan). + +## Knobs +- **`taskKey`** — the scope of learning. Use a stable key per task *type* (e.g. `"answer-support-question"`), not per individual input — you want lessons to generalise across instances, not pile up per question. Different keys never share lessons. +- **`recallK`** (default ~3) — how many lessons/skills to inject. Too many bloats the prompt and drowns the current task; 2–4 is plenty. +- **`skillThreshold`** (default ~3) — how often a sequence must succeed before it's a "skill". Lower ⇒ faster skill formation but more noise; higher ⇒ only robust patterns. +- **Store growth** — `getStats()` / `tierCounts()` to watch size; HNSW + `quantization: 'rabitq'` (`coarseThenRerank`) keep recall fast on big stores; `promote`/`evictTier` to manage tiers (keep hot lessons in `working`, age out stale ones from `short`). + +## Inspecting +`/dspy-reflexion ` shows lessons (`lessonsText`/`recall`), promoted skills (`getSkills`), and episode history. Run `/dspy-react-run` a few times on a hard task and watch lessons appear, then a skill once the agent finds a sequence that keeps working. + +## Caution +Reflexion is only as good as the `success` signal. If a "successful" run is actually wrong (weak metric / no metric), you'll promote bad skills. Make `success` mean *correct*, not *finished*. diff --git a/plugins/dspy-react/skills/tool-design/SKILL.md b/plugins/dspy-react/skills/tool-design/SKILL.md new file mode 100644 index 0000000..f4c0244 --- /dev/null +++ b/plugins/dspy-react/skills/tool-design/SKILL.md @@ -0,0 +1,24 @@ +--- +name: tool-design +version: "0.1.0" +author: rUv +tags: [dspy, react, tools, agent, tool-use] +description: > + How to design the tool registry for a DSPy.ts ReAct agent — names, descriptions, argument shapes, error handling — so the model picks the right tool and recovers from failures. + Use when: writing or debugging the tools of a ReAct program (`/dspy-react-new`, `/dspy-react-run`). +--- +# Tool design for ReAct + +In `dspy.ts`, a ReAct tool is `{ name, description, handler: async (args) => observation }`. The model never sees your code — only `name` + `description`. Treat the description as the API doc the model reads at every step. + +## Rules of thumb +- **The description must answer three things:** *what does it do*, *what args does it take* (names + types + an example), and *when to use it vs the other tools*. "Searches the web" is useless; "search(query: string) — full-text web search; returns the top 5 result snippets. Use for current facts you don't know. Not for arithmetic (use `calc`) or for reading a known URL (use `fetch`)." is usable. +- **Keep the set small (3–6).** Every extra tool is another thing the model can pick wrong. If two tools overlap, merge them or sharpen the "when to use" boundary. +- **Name tools for the action**, lowercase, verb-ish: `search`, `calc`, `fetch_url`, `lookup_order`. Not `tool1`, not `WebSearchAPIv2`. +- **Args: flat and named.** A single object with a couple of clearly-named fields beats positional or deeply nested args. Validate inside the handler; on bad args, return an observation that says what was wrong ("error: `query` is required") — don't throw. +- **Handlers must not crash the loop.** A thrown error becomes the observation; the agent then has to reason about a stack trace. Catch, and return a short, actionable string instead ("error: order #123 not found — check the id"). +- **Make observations terse and parseable.** The whole observation goes back into the context every step. Return the answer, not a 5KB JSON blob; truncate, summarise, or paginate. +- **Idempotent / read-only where possible.** ReAct may call a tool more than once (retry, re-check). Side-effectful tools (sending mail, writing data) should be obviously named and ideally require a confirmation arg. + +## Step budget +`maxSteps` 4–8. If the agent regularly hits the cap, don't raise it — the tools are ambiguous or the thought prompt doesn't explain the task. Read a `/dspy-react-run` trace: the step where it picked the wrong tool tells you which description to fix.