diff --git a/.gitignore b/.gitignore index b5d3257e7e618..48218130a50d7 100644 --- a/.gitignore +++ b/.gitignore @@ -27,7 +27,6 @@ mise.toml apps/android/.gradle/ apps/android/app/build/ apps/android/.cxx/ - # Bun build artifacts *.bun-build apps/macos/.build/ @@ -49,6 +48,8 @@ apps/ios/.swiftpm/ apps/ios/.derivedData/ apps/ios/.local-signing.xcconfig vendor/ +!src/auto-reply/reply/export-html/vendor/ +!src/auto-reply/reply/export-html/vendor/** apps/ios/Clawdbot.xcodeproj/ apps/ios/Clawdbot.xcodeproj/** apps/macos/.build/** @@ -79,6 +80,7 @@ apps/ios/*.mobileprovision # Local untracked files .local/ +.cursor/ docs/.local/ IDENTITY.md USER.md @@ -88,6 +90,8 @@ USER.md # local tooling .serena/ +# Rust build artifacts (RustFS) +packages/rustfs/target/ # Agent credentials and memory (NEVER COMMIT) /memory/ .agent/*.json @@ -95,8 +99,6 @@ USER.md /local/ package-lock.json .claude/settings.local.json -.agents/ -.agents .agent/ skills-lock.json diff --git a/.oxfmtrc.jsonc b/.oxfmtrc.jsonc index 0a928d5f9bae1..ac79efd9134cc 100644 --- a/.oxfmtrc.jsonc +++ b/.oxfmtrc.jsonc @@ -18,6 +18,7 @@ "node_modules/", "patches/", "pnpm-lock.yaml/", + "packages/rustfs/target/", "src/gateway/server-methods/CLAUDE.md", "src/auto-reply/reply/export-html/", "Swabble/", diff --git a/deploy/k8s/deep-memory/deep-memory-server.yaml b/deploy/k8s/deep-memory/deep-memory-server.yaml new file mode 100644 index 0000000000000..33649e62ddf95 --- /dev/null +++ b/deploy/k8s/deep-memory/deep-memory-server.yaml @@ -0,0 +1,135 @@ +apiVersion: v1 +kind: Service +metadata: + name: deep-memory-server + labels: + app.kubernetes.io/name: deep-memory-server +spec: + selector: + app.kubernetes.io/name: deep-memory-server + ports: + - name: http + port: 8088 + targetPort: 8088 +--- +apiVersion: v1 +kind: Secret +metadata: + name: deep-memory-server-secrets +type: Opaque +stringData: + # REQUIRED in production: configure API keys and limit namespaces. + # Example: + # [ + # { "key": "", "role": "read", "namespaces": ["teamA"] }, + # { "key": "", "role": "write", "namespaces": ["teamA"] }, + # { "key": "", "role": "admin", "namespaces": ["teamA"] } + # ] + API_KEYS_JSON: "[]" + # Optional: set if Qdrant is secured. + QDRANT_API_KEY: "" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: deep-memory-server-config +data: + PORT: "8088" + HOST: "0.0.0.0" + REQUIRE_API_KEY: "true" + # In Kubernetes, prefer keeping /metrics behind NetworkPolicy; leave this false by default. + ALLOW_UNAUTHENTICATED_METRICS: "false" + RATE_LIMIT_ENABLED: "true" + RATE_LIMIT_WINDOW_MS: "60000" + RATE_LIMIT_RETRIEVE_PER_WINDOW: "300" + RATE_LIMIT_UPDATE_PER_WINDOW: "60" + RATE_LIMIT_FORGET_PER_WINDOW: "10" + RATE_LIMIT_QUEUE_ADMIN_PER_WINDOW: "60" + UPDATE_BACKLOG_REJECT_PENDING: "2000" + UPDATE_BACKLOG_RETRY_AFTER_SECONDS: "30" + + QDRANT_URL: "http://qdrant:6333" + QDRANT_COLLECTION: "openclaw_memories" + VECTOR_DIMS: "384" + + NEO4J_URI: "bolt://neo4j:7687" + NEO4J_USER: "neo4j" + + LOG_LEVEL: "info" + UPDATE_CONCURRENCY: "1" + QUEUE_DIR: "/data/queue" + AUDIT_LOG_PATH: "/data/audit/audit.jsonl" + EMBEDDING_MODEL: "Xenova/bge-small-en-v1.5" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deep-memory-server + labels: + app.kubernetes.io/name: deep-memory-server +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: deep-memory-server + template: + metadata: + labels: + app.kubernetes.io/name: deep-memory-server + spec: + containers: + - name: deep-memory-server + # Build and push your own image from packages/deep-memory-server/Dockerfile + # then set this to your registry. + image: "openclaw/deep-memory-server:local" + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8088 + envFrom: + - configMapRef: + name: deep-memory-server-config + - secretRef: + name: deep-memory-server-secrets + env: + - name: NEO4J_PASSWORD + valueFrom: + secretKeyRef: + name: neo4j-auth + key: password + volumeMounts: + - name: data + mountPath: /data + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi + readinessProbe: + httpGet: + path: /readyz + port: 8088 + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: 8088 + initialDelaySeconds: 10 + periodSeconds: 20 + volumes: + - name: data + persistentVolumeClaim: + claimName: deep-memory-server-data +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: deep-memory-server-data +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 5Gi diff --git a/deploy/k8s/deep-memory/kustomization.yaml b/deploy/k8s/deep-memory/kustomization.yaml new file mode 100644 index 0000000000000..9fc962f5ae665 --- /dev/null +++ b/deploy/k8s/deep-memory/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: openclaw + +resources: + - namespace.yaml + - qdrant.yaml + - neo4j.yaml + - deep-memory-server.yaml + - networkpolicy.yaml diff --git a/deploy/k8s/deep-memory/namespace.yaml b/deploy/k8s/deep-memory/namespace.yaml new file mode 100644 index 0000000000000..394432ee0d332 --- /dev/null +++ b/deploy/k8s/deep-memory/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: openclaw diff --git a/deploy/k8s/deep-memory/neo4j.yaml b/deploy/k8s/deep-memory/neo4j.yaml new file mode 100644 index 0000000000000..4a14a0ea6f056 --- /dev/null +++ b/deploy/k8s/deep-memory/neo4j.yaml @@ -0,0 +1,95 @@ +apiVersion: v1 +kind: Service +metadata: + name: neo4j + labels: + app.kubernetes.io/name: neo4j +spec: + selector: + app.kubernetes.io/name: neo4j + ports: + - name: http + port: 7474 + targetPort: 7474 + - name: bolt + port: 7687 + targetPort: 7687 +--- +apiVersion: v1 +kind: Secret +metadata: + name: neo4j-auth +type: Opaque +stringData: + # Set this to a strong password before applying. + password: "openclaw" +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: neo4j + labels: + app.kubernetes.io/name: neo4j +spec: + serviceName: neo4j + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: neo4j + template: + metadata: + labels: + app.kubernetes.io/name: neo4j + spec: + containers: + - name: neo4j + image: neo4j:5.26 + ports: + - name: http + containerPort: 7474 + - name: bolt + containerPort: 7687 + env: + - name: NEO4J_AUTH + value: "neo4j/$(NEO4J_PASSWORD)" + - name: NEO4J_PASSWORD + valueFrom: + secretKeyRef: + name: neo4j-auth + key: password + - name: NEO4J_server_memory_pagecache_size + value: "512M" + - name: NEO4J_server_memory_heap_initial__size + value: "512M" + - name: NEO4J_server_memory_heap_max__size + value: "512M" + volumeMounts: + - name: data + mountPath: /data + resources: + requests: + cpu: 200m + memory: 1Gi + limits: + cpu: "2" + memory: 2Gi + readinessProbe: + httpGet: + path: / + port: 7474 + initialDelaySeconds: 15 + periodSeconds: 10 + livenessProbe: + httpGet: + path: / + port: 7474 + initialDelaySeconds: 30 + periodSeconds: 20 + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 20Gi diff --git a/deploy/k8s/deep-memory/networkpolicy.yaml b/deploy/k8s/deep-memory/networkpolicy.yaml new file mode 100644 index 0000000000000..62bb3d303d67b --- /dev/null +++ b/deploy/k8s/deep-memory/networkpolicy.yaml @@ -0,0 +1,17 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: deep-memory-server-ingress +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: deep-memory-server + policyTypes: + - Ingress + ingress: + # Allow in-namespace callers by default. Tighten this to specific pods/namespaces + # in production (e.g. OpenClaw gateway pods + Prometheus). + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: openclaw diff --git a/deploy/k8s/deep-memory/qdrant.yaml b/deploy/k8s/deep-memory/qdrant.yaml new file mode 100644 index 0000000000000..771e20fdd0d5b --- /dev/null +++ b/deploy/k8s/deep-memory/qdrant.yaml @@ -0,0 +1,67 @@ +apiVersion: v1 +kind: Service +metadata: + name: qdrant + labels: + app.kubernetes.io/name: qdrant +spec: + selector: + app.kubernetes.io/name: qdrant + ports: + - name: http + port: 6333 + targetPort: 6333 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: qdrant + labels: + app.kubernetes.io/name: qdrant +spec: + serviceName: qdrant + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: qdrant + template: + metadata: + labels: + app.kubernetes.io/name: qdrant + spec: + containers: + - name: qdrant + image: qdrant/qdrant:v1.13.4 + ports: + - name: http + containerPort: 6333 + volumeMounts: + - name: data + mountPath: /qdrant/storage + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + readinessProbe: + httpGet: + path: / + port: 6333 + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: / + port: 6333 + initialDelaySeconds: 10 + periodSeconds: 20 + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 10Gi diff --git a/deploy/prometheus/deep-memory.rules.yml b/deploy/prometheus/deep-memory.rules.yml new file mode 100644 index 0000000000000..b28e6aee2b1ec --- /dev/null +++ b/deploy/prometheus/deep-memory.rules.yml @@ -0,0 +1,59 @@ +groups: + - name: deep-memory-server + rules: + - alert: DeepMemoryServerHighErrorRate + expr: | + ( + sum(rate(deep_memory_http_requests_total{status=~"5.."}[5m])) + / + clamp_min(sum(rate(deep_memory_http_requests_total[5m])), 1e-9) + ) > 0.02 + for: 10m + labels: + severity: page + annotations: + summary: "deep-memory-server 5xx error rate is high" + description: "5xx rate > 2% for 10m. Check /readyz, DB availability, and queue backlog." + + - alert: DeepMemoryServerHighP95Latency + expr: | + histogram_quantile( + 0.95, + sum by (le) (rate(deep_memory_http_request_duration_seconds_bucket[5m])) + ) > 1 + for: 10m + labels: + severity: page + annotations: + summary: "deep-memory-server p95 latency is high" + description: "p95 request latency > 1s for 10m. Check DB latency, CPU, and queue saturation." + + - alert: DeepMemoryQueueBacklogHigh + expr: deep_memory_queue_pending > 1000 + for: 15m + labels: + severity: page + annotations: + summary: "deep-memory queue backlog is high" + description: "Pending update tasks > 1000 for 15m. Consider scaling UPDATE_CONCURRENCY, checking DB health, or throttling updates." + + - alert: DeepMemoryQueueStuckNoWorkers + expr: deep_memory_queue_pending > 0 and deep_memory_queue_active == 0 + for: 10m + labels: + severity: ticket + annotations: + summary: "deep-memory queue appears stuck" + description: "There are pending tasks but no active workers for 10m. Check logs and whether the process is running." + +# Optional: if you use a blackbox exporter to probe /readyz, add an alert like: +# +# - alert: DeepMemoryReadyzFailing +# expr: probe_success{job="deep-memory-readyz"} == 0 +# for: 5m +# labels: +# severity: page +# annotations: +# summary: "deep-memory /readyz probe failing" +# description: "Blackbox probe reports failures for /readyz." + diff --git a/docker-compose.yml b/docker-compose.yml index 614a1f8d53348..a2736278a7608 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,21 +1,24 @@ services: openclaw-gateway: image: ${OPENCLAW_IMAGE:-openclaw:local} + build: . + restart: unless-stopped + env_file: + - .env environment: - HOME: /home/node - TERM: xterm-256color - OPENCLAW_GATEWAY_TOKEN: ${OPENCLAW_GATEWAY_TOKEN} - CLAUDE_AI_SESSION_KEY: ${CLAUDE_AI_SESSION_KEY} - CLAUDE_WEB_SESSION_KEY: ${CLAUDE_WEB_SESSION_KEY} - CLAUDE_WEB_COOKIE: ${CLAUDE_WEB_COOKIE} - volumes: - - ${OPENCLAW_CONFIG_DIR}:/home/node/.openclaw - - ${OPENCLAW_WORKSPACE_DIR}:/home/node/.openclaw/workspace + - HOME=/home/node + - NODE_ENV=production + - TERM=xterm-256color + - OPENCLAW_GATEWAY_BIND=${OPENCLAW_GATEWAY_BIND:-lan} + - OPENCLAW_GATEWAY_PORT=${OPENCLAW_GATEWAY_PORT:-18789} + - OPENCLAW_GATEWAY_TOKEN=${OPENCLAW_GATEWAY_TOKEN:-} + - XDG_CONFIG_HOME=/home/node/.openclaw + volumes: + - ${OPENCLAW_CONFIG_DIR:-$HOME/.openclaw}:/home/node/.openclaw + - ${OPENCLAW_WORKSPACE_DIR:-$HOME/.openclaw/workspace}:/home/node/.openclaw/workspace ports: - "${OPENCLAW_GATEWAY_PORT:-18789}:18789" - "${OPENCLAW_BRIDGE_PORT:-18790}:18790" - init: true - restart: unless-stopped command: [ "node", @@ -24,23 +27,211 @@ services: "--bind", "${OPENCLAW_GATEWAY_BIND:-lan}", "--port", - "18789", + "${OPENCLAW_GATEWAY_PORT:-18789}", ] openclaw-cli: image: ${OPENCLAW_IMAGE:-openclaw:local} + build: . + restart: "no" + env_file: + - .env environment: - HOME: /home/node - TERM: xterm-256color - OPENCLAW_GATEWAY_TOKEN: ${OPENCLAW_GATEWAY_TOKEN} - BROWSER: echo - CLAUDE_AI_SESSION_KEY: ${CLAUDE_AI_SESSION_KEY} - CLAUDE_WEB_SESSION_KEY: ${CLAUDE_WEB_SESSION_KEY} - CLAUDE_WEB_COOKIE: ${CLAUDE_WEB_COOKIE} - volumes: - - ${OPENCLAW_CONFIG_DIR}:/home/node/.openclaw - - ${OPENCLAW_WORKSPACE_DIR}:/home/node/.openclaw/workspace - stdin_open: true - tty: true - init: true + - HOME=/home/node + - NODE_ENV=production + - TERM=xterm-256color + - OPENCLAW_GATEWAY_BIND=${OPENCLAW_GATEWAY_BIND:-lan} + - OPENCLAW_GATEWAY_PORT=${OPENCLAW_GATEWAY_PORT:-18789} + - OPENCLAW_GATEWAY_TOKEN=${OPENCLAW_GATEWAY_TOKEN:-} + - XDG_CONFIG_HOME=/home/node/.openclaw + volumes: + - ${OPENCLAW_CONFIG_DIR:-$HOME/.openclaw}:/home/node/.openclaw + - ${OPENCLAW_WORKSPACE_DIR:-$HOME/.openclaw/workspace}:/home/node/.openclaw/workspace entrypoint: ["node", "dist/index.js"] + + # Deep memory stack (optional). Enable via: docker compose --profile deep-memory up + qdrant: + profiles: ["deep-memory", "deep-memory-prod"] + # Keep reasonably close to @qdrant/js-client-rest minor version (compat check is strict). + image: qdrant/qdrant:v1.16.2 + ports: + - "${OPENCLAW_QDRANT_PORT:-6333}:6333" + volumes: + - qdrant_data:/qdrant/storage + restart: unless-stopped + + neo4j: + profiles: ["deep-memory", "deep-memory-prod"] + image: neo4j:5.26 + environment: + NEO4J_AUTH: "neo4j/${OPENCLAW_NEO4J_PASSWORD:-openclaw}" + NEO4J_server_memory_pagecache_size: "512M" + NEO4J_server_memory_heap_initial__size: "512M" + NEO4J_server_memory_heap_max__size: "512M" + ports: + - "${OPENCLAW_NEO4J_HTTP_PORT:-7474}:7474" + - "${OPENCLAW_NEO4J_BOLT_PORT:-7687}:7687" + volumes: + - neo4j_data:/data + restart: unless-stopped + + deep-memory-server: + profiles: ["deep-memory"] + image: node:22-alpine + working_dir: /app + environment: + PORT: 8088 + HOST: 0.0.0.0 + REQUIRE_API_KEY: "${OPENCLAW_DEEPMEM_REQUIRE_API_KEY:-false}" + API_KEYS_JSON: "${OPENCLAW_DEEPMEM_API_KEYS_JSON:-}" + QDRANT_URL: "http://qdrant:6333" + QDRANT_COLLECTION: "${OPENCLAW_DEEPMEM_COLLECTION:-openclaw_memories}" + VECTOR_DIMS: "${OPENCLAW_DEEPMEM_DIMS:-768}" + MIN_SEMANTIC_SCORE: "${OPENCLAW_DEEPMEM_MIN_SEMANTIC_SCORE:-0.6}" + NEO4J_URI: "bolt://neo4j:7687" + NEO4J_USER: "neo4j" + NEO4J_PASSWORD: "${OPENCLAW_NEO4J_PASSWORD:-openclaw}" + LOG_LEVEL: "${OPENCLAW_DEEPMEM_LOG_LEVEL:-info}" + RETRIEVE_CACHE_TTL_MS: "${OPENCLAW_DEEPMEM_CACHE_TTL_MS:-300000}" + RETRIEVE_CACHE_MAX: "${OPENCLAW_DEEPMEM_CACHE_MAX:-500}" + UPDATE_CONCURRENCY: "${OPENCLAW_DEEPMEM_UPDATE_CONCURRENCY:-1}" + EMBEDDING_MODEL: "${OPENCLAW_DEEPMEM_EMBEDDING_MODEL:-Xenova/bge-base-zh-v1.5}" + volumes: + - ./:/app + command: + [ + "sh", + "-lc", + "corepack enable && pnpm -s install && pnpm -s --dir packages/deep-memory-server dev", + ] + ports: + - "${OPENCLAW_DEEPMEM_PORT:-8088}:8088" + depends_on: + - qdrant + - neo4j + restart: unless-stopped + + # Production-oriented deep-memory-server (built image + persistent queue dir). + # Enable via: docker compose --profile deep-memory-prod up -d + deep-memory-server-prod: + profiles: ["deep-memory-prod"] + build: + context: . + dockerfile: packages/deep-memory-server/Dockerfile + environment: + PORT: 8088 + HOST: 0.0.0.0 + # For local dev, default to no API key. Set OPENCLAW_DEEPMEM_REQUIRE_API_KEY=true + API_KEYS_JSON to lock down. + REQUIRE_API_KEY: "${OPENCLAW_DEEPMEM_REQUIRE_API_KEY:-false}" + API_KEYS_JSON: "${OPENCLAW_DEEPMEM_API_KEYS_JSON:-}" + RATE_LIMIT_ENABLED: "${OPENCLAW_DEEPMEM_RATE_LIMIT_ENABLED:-true}" + RATE_LIMIT_WINDOW_MS: "${OPENCLAW_DEEPMEM_RATE_LIMIT_WINDOW_MS:-60000}" + RATE_LIMIT_RETRIEVE_PER_WINDOW: "${OPENCLAW_DEEPMEM_RL_RETRIEVE:-300}" + RATE_LIMIT_UPDATE_PER_WINDOW: "${OPENCLAW_DEEPMEM_RL_UPDATE:-60}" + RATE_LIMIT_FORGET_PER_WINDOW: "${OPENCLAW_DEEPMEM_RL_FORGET:-10}" + RATE_LIMIT_QUEUE_ADMIN_PER_WINDOW: "${OPENCLAW_DEEPMEM_RL_QUEUE_ADMIN:-60}" + UPDATE_BACKLOG_REJECT_PENDING: "${OPENCLAW_DEEPMEM_BACKLOG_REJECT_PENDING:-2000}" + UPDATE_BACKLOG_RETRY_AFTER_SECONDS: "${OPENCLAW_DEEPMEM_BACKLOG_RETRY_AFTER_SECONDS:-30}" + QDRANT_URL: "http://qdrant:6333" + QDRANT_COLLECTION: "${OPENCLAW_DEEPMEM_COLLECTION:-openclaw_memories}" + VECTOR_DIMS: "${OPENCLAW_DEEPMEM_DIMS:-768}" + MIN_SEMANTIC_SCORE: "${OPENCLAW_DEEPMEM_MIN_SEMANTIC_SCORE:-0.6}" + NEO4J_URI: "bolt://neo4j:7687" + NEO4J_USER: "neo4j" + NEO4J_PASSWORD: "${OPENCLAW_NEO4J_PASSWORD:-openclaw}" + LOG_LEVEL: "${OPENCLAW_DEEPMEM_LOG_LEVEL:-info}" + RETRIEVE_CACHE_TTL_MS: "${OPENCLAW_DEEPMEM_CACHE_TTL_MS:-300000}" + RETRIEVE_CACHE_MAX: "${OPENCLAW_DEEPMEM_CACHE_MAX:-500}" + UPDATE_CONCURRENCY: "${OPENCLAW_DEEPMEM_UPDATE_CONCURRENCY:-1}" + EMBEDDING_MODEL: "${OPENCLAW_DEEPMEM_EMBEDDING_MODEL:-Xenova/bge-base-zh-v1.5}" + QUEUE_DIR: "/data/queue" + AUDIT_LOG_PATH: "/data/audit/audit.jsonl" + volumes: + - deepmem_data:/data + ports: + - "${OPENCLAW_DEEPMEM_PORT:-8088}:8088" + depends_on: + - qdrant + - neo4j + healthcheck: + test: + [ + "CMD", + "node", + "-e", + "fetch('http://127.0.0.1:8088/readyz').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))", + ] + interval: 10s + timeout: 3s + retries: 5 + restart: unless-stopped + + # RustFS (optional). Enable via: docker compose --profile rustfs up -d + rustfs: + profiles: ["rustfs"] + build: + context: . + dockerfile: packages/rustfs/Dockerfile + environment: + RUSTFS_PORT: 8099 + RUSTFS_DATA_DIR: /data + RUSTFS_DB_PATH: /data/meta.db + # For local dev, default to no API key. Set OPENCLAW_RUSTFS_REQUIRE_API_KEY=true + RUSTFS_API_KEYS_JSON to lock down. + RUSTFS_REQUIRE_API_KEY: "${OPENCLAW_RUSTFS_REQUIRE_API_KEY:-false}" + RUSTFS_API_KEYS_JSON: "${OPENCLAW_RUSTFS_API_KEYS_JSON:-}" + # Optional: enables encryption at rest for newly ingested files + RUSTFS_MASTER_KEY: "${OPENCLAW_RUSTFS_MASTER_KEY:-}" + # Required for public share links (HMAC signing). Keep this secret. + RUSTFS_SIGNING_KEY: "${OPENCLAW_RUSTFS_SIGNING_KEY:-}" + # Optional base URL used when returning share links (otherwise returns a relative path). + RUSTFS_PUBLIC_BASE_URL: "${OPENCLAW_RUSTFS_PUBLIC_BASE_URL:-}" + # Optional audit log (JSONL) for sensitive actions. + RUSTFS_AUDIT_LOG_PATH: "${OPENCLAW_RUSTFS_AUDIT_LOG_PATH:-/data/audit/audit.jsonl}" + ports: + - "${OPENCLAW_RUSTFS_PORT:-8099}:8099" + volumes: + - rustfs_data:/data + healthcheck: + test: ["CMD", "sh", "-lc", "curl -fsS http://127.0.0.1:8099/readyz >/dev/null 2>&1"] + interval: 10s + timeout: 3s + retries: 5 + restart: unless-stopped + + # RustFS worker (optional). Enable via: + # docker compose --profile rustfs --profile deep-memory-prod --profile rustfs-worker up -d + rustfs-worker: + profiles: ["rustfs-worker"] + build: + context: . + dockerfile: packages/rustfs-worker/Dockerfile + environment: + RUSTFS_BASE_URL: "http://rustfs:8099" + RUSTFS_API_KEY: "${OPENCLAW_RUSTFS_WORKER_API_KEY:-}" + RUSTFS_POLL_INTERVAL_MS: "${OPENCLAW_RUSTFS_WORKER_POLL_INTERVAL_MS:-5000}" + RUSTFS_LEASE_MS: "${OPENCLAW_RUSTFS_WORKER_LEASE_MS:-300000}" + RUSTFS_PENDING_LIMIT: "${OPENCLAW_RUSTFS_WORKER_PENDING_LIMIT:-25}" + RUSTFS_MAX_DOWNLOAD_BYTES: "${OPENCLAW_RUSTFS_WORKER_MAX_DOWNLOAD_BYTES:-2097152}" + DEEP_MEMORY_BASE_URL: "http://deep-memory-server-prod:8088" + DEEP_MEMORY_API_KEY: "${OPENCLAW_DEEPMEM_WORKER_API_KEY:-}" + DEEP_MEMORY_NAMESPACE: "${OPENCLAW_DEEPMEM_NAMESPACE:-}" + DEEP_MEMORY_ASYNC: "${OPENCLAW_DEEPMEM_WORKER_ASYNC:-true}" + DEEP_MEMORY_TIMEOUT_MS: "${OPENCLAW_DEEPMEM_WORKER_TIMEOUT_MS:-30000}" + DEEP_MEMORY_CHUNK_MAX_CHARS: "${OPENCLAW_DEEPMEM_WORKER_CHUNK_MAX_CHARS:-3500}" + DEEP_MEMORY_CHUNK_OVERLAP_CHARS: "${OPENCLAW_DEEPMEM_WORKER_CHUNK_OVERLAP_CHARS:-200}" + DEEP_MEMORY_BACKOFF_MS: "${OPENCLAW_DEEPMEM_WORKER_BACKOFF_MS:-10000}" + WORKER_STATE_PATH: "${OPENCLAW_RUSTFS_WORKER_STATE_PATH:-/data/worker/state.json}" + depends_on: + rustfs: + condition: service_healthy + deep-memory-server-prod: + condition: service_healthy + volumes: + - rustfs_data:/data + restart: unless-stopped + +volumes: + qdrant_data: + neo4j_data: + deepmem_data: + rustfs_data: diff --git a/docs/concepts/deep-memory.md b/docs/concepts/deep-memory.md new file mode 100644 index 0000000000000..a5ed5863c10f7 --- /dev/null +++ b/docs/concepts/deep-memory.md @@ -0,0 +1,59 @@ +--- +summary: "Deep memory: external long-term memory service (Qdrant + Neo4j) for OpenClaw" +read_when: + - You want cross-session memory and long-term recall beyond Markdown memory files + - You are deploying deep-memory-server in production +title: "Deep memory" +--- + +# Deep memory + +OpenClaw supports an optional **deep memory** backend implemented as an external service: + +- **deep-memory-server**: HTTP API used by OpenClaw for retrieval + ingestion +- **Qdrant**: vector search (semantic recall) +- **Neo4j**: relationship graph (entity/topic/event links + RELATED_TO) + +This is separate from the default Markdown-based memory files. See [Memory](/concepts/memory) for the built-in model. + +## What it does + +- **Online path** (every turn): OpenClaw calls `POST /retrieve_context` and injects a read-only memory block into the model prompt. +- **Offline path** (async): OpenClaw sends transcript batches to `POST /update_memory_index` (usually async) so deep memories can be extracted, deduped, and linked. +- **Admin operations**: `POST /forget` deletes by session or id (best-effort dual-store). + +## Production deployment (Docker Compose) + +The repo root `docker-compose.yml` includes two profiles: + +- `deep-memory`: dev-friendly (runs from the repo, installs deps at container startup) +- `deep-memory-prod`: production-oriented (built image + persistent queue/audit volumes) + +Example: + +```bash +docker compose --profile deep-memory-prod up -d +``` + +## Observability / probes + +- `GET /health`: liveness + queue/config guardrail summary +- `GET /health/details`: detailed health (includes dependency address summary; admin-only when API keys are required) +- `GET /readyz`: readiness (Qdrant + Neo4j) +- `GET /metrics`: Prometheus metrics (admin-only when API keys are required) + +For production operations and monitoring, see: + +- [Deep memory ops](/reference/deep-memory-ops) +- [Deep memory server reference](/reference/deep-memory-server) +- [Deep memory alerting](/reference/deep-memory-alerting) +- [Deep memory on Kubernetes](/reference/deep-memory-kubernetes) + +## Security checklist + +- Set `REQUIRE_API_KEY=true` +- Configure `API_KEYS_JSON` and restrict: + - `read` for `/retrieve_context` + - `write` for `/update_memory_index` + - `admin` for `/forget`, `/queue/*`, `/metrics` +- Run behind TLS (reverse proxy) and keep the service private to your gateway network diff --git a/docs/concepts/memory.md b/docs/concepts/memory.md index c8b2db0b091c8..9acc8b0448df6 100644 --- a/docs/concepts/memory.md +++ b/docs/concepts/memory.md @@ -14,6 +14,9 @@ source of truth; the model only "remembers" what gets written to disk. Memory search tools are provided by the active memory plugin (default: `memory-core`). Disable memory plugins with `plugins.slots.memory = "none"`. +OpenClaw also supports an optional **deep memory** backend (external service with +vector + graph storage) for cross-session recall. See [Deep memory](/concepts/deep-memory). + ## Memory files (Markdown) The default workspace layout uses two memory layers: diff --git a/docs/reference/deep-memory-alerting.md b/docs/reference/deep-memory-alerting.md new file mode 100644 index 0000000000000..9e0561fb1eb57 --- /dev/null +++ b/docs/reference/deep-memory-alerting.md @@ -0,0 +1,46 @@ +--- +summary: "Prometheus alerting for deep-memory-server (recommended rules and dashboards)" +read_when: + - You are monitoring deep-memory-server in production + - You want recommended alerts for errors, latency, and queue backlog +title: "Deep memory alerting" +--- + +# Deep memory alerting + +deep-memory-server exposes Prometheus metrics at `GET /metrics`. + +In production, `/metrics` should be protected (admin-only when API keys are required). If you run without API keys, metrics are intentionally disabled by default unless you set `ALLOW_UNAUTHENTICATED_METRICS=true`. + +## Metrics to scrape + +Scrape deep-memory-server: + +- `deep_memory_http_requests_total` +- `deep_memory_http_request_duration_seconds` (histogram) +- `deep_memory_queue_pending` +- `deep_memory_queue_active` +- `deep_memory_queue_inflight_keys` + +## Recommended alert rules + +This repo includes a starter Prometheus rules file: + +- `deploy/prometheus/deep-memory.rules.yml` + +Import it into your Prometheus (or convert to a `PrometheusRule` CRD for kube-prometheus-stack). + +### Suggested thresholds + +You should tune thresholds to your expected load, but these defaults are usually a good starting point: + +- **5xx error rate**: page at > 2% for 10m +- **p95 latency**: page at > 1s for 10m +- **queue backlog**: page at pending > 1000 for 15m + +## Readiness probing + +Prometheus itself does not inspect HTTP status codes from scrapes. For readiness: + +- Use a blackbox exporter to probe `GET /readyz` and alert on failures (`probe_success == 0`). +- Alternatively, have your orchestrator (Compose/Kubernetes) gate traffic on readiness probes. diff --git a/docs/reference/deep-memory-kubernetes.md b/docs/reference/deep-memory-kubernetes.md new file mode 100644 index 0000000000000..530e0fa3e3f9f --- /dev/null +++ b/docs/reference/deep-memory-kubernetes.md @@ -0,0 +1,75 @@ +--- +summary: "Deploy deep-memory-server (Qdrant + Neo4j) to Kubernetes" +read_when: + - You are deploying deep memory in Kubernetes + - You want recommended probes, secrets, and network policies +title: "Deep memory on Kubernetes" +--- + +# Deep memory on Kubernetes + +This repo includes a Kustomize base that deploys: + +- Qdrant (StatefulSet + PVC) +- Neo4j (StatefulSet + PVC) +- deep-memory-server (Deployment + PVC for queue/audit data) + +Manifests live under: + +- `deploy/k8s/deep-memory` + +## Apply (Kustomize) + +```bash +kubectl apply -k deploy/k8s/deep-memory +``` + +The base uses namespace `openclaw`. + +## Important: set secrets before production use + +Edit these resources before applying in production: + +- `deploy/k8s/deep-memory/neo4j.yaml` (`neo4j-auth` Secret password) +- `deploy/k8s/deep-memory/deep-memory-server.yaml` (`deep-memory-server-secrets`): + - `API_KEYS_JSON` (required) + - `QDRANT_API_KEY` (optional, if you secure Qdrant) + +## Probes + +deep-memory-server uses: + +- readiness: `GET /readyz` +- liveness: `GET /health` + +These endpoints are designed for orchestration. + +## Metrics scraping + +`GET /metrics` is: + +- admin-only when API keys are required +- intentionally disabled by default when running without API keys, unless `ALLOW_UNAUTHENTICATED_METRICS=true` + +In Kubernetes, the recommended pattern is: + +1. Keep `ALLOW_UNAUTHENTICATED_METRICS=false` +2. Allow Prometheus to reach deep-memory-server via NetworkPolicy +3. Use an in-cluster scrape config that injects the required auth (if your Prometheus supports it) + +If your Prometheus cannot inject `x-api-key`, you can temporarily set: + +- `ALLOW_UNAUTHENTICATED_METRICS=true` + +and rely on **NetworkPolicy** + cluster RBAC boundaries to restrict access to `/metrics`. + +## NetworkPolicy + +The base includes a minimal NetworkPolicy: + +- `deploy/k8s/deep-memory/networkpolicy.yaml` + +It currently allows in-namespace ingress. Tighten it to only: + +- OpenClaw gateway pods +- Prometheus pods (if scraping metrics) diff --git a/docs/reference/deep-memory-ops.md b/docs/reference/deep-memory-ops.md new file mode 100644 index 0000000000000..1dd8c2fedb274 --- /dev/null +++ b/docs/reference/deep-memory-ops.md @@ -0,0 +1,309 @@ +--- +summary: "Production operations for deep-memory-server: backups, restores, upgrades, and security" +read_when: + - You are running deep-memory-server in production + - You need backup/restore procedures or an upgrade checklist +title: "Deep memory ops" +--- + +# Deep memory ops + +This page is a production-focused runbook for OpenClaw's deep memory stack: + +- `deep-memory-server` (stateless API + durable queue directory) +- Qdrant (vector DB) +- Neo4j (graph DB) + +It assumes you are using the repo root `docker-compose.yml` profiles: + +- `deep-memory-prod`: built image + persistent `deepmem_data` volume +- `deep-memory`: dev-friendly profile (not recommended for production) + +For configuration and API contract details, see [Deep memory server reference](/reference/deep-memory-server). + +## Quick health checks + +From the host running the services: + +- `GET http://127.0.0.1:8088/health` (liveness + queue stats) +- `GET http://127.0.0.1:8088/health/details` (detailed health + schema checks; admin-only when API keys are required) +- `GET http://127.0.0.1:8088/readyz` (readiness; returns 503 if Qdrant/Neo4j are unavailable) + +If you scrape metrics (recommended): + +- `GET http://127.0.0.1:8088/metrics` (admin-only when API keys are required) + +## Required security settings (production) + +Deep memory is a powerful write/delete surface. For production: + +- Set `REQUIRE_API_KEY=true` +- Configure `API_KEYS_JSON` with separate roles: + - `read`: `/retrieve_context` + - `write`: `/update_memory_index` + - `admin`: `/forget`, `/queue/*`, `/metrics` + +Example `API_KEYS_JSON` (use your own random secrets): + +```json +[ + { "key": "", "role": "read", "namespaces": ["teamA"] }, + { "key": "", "role": "write", "namespaces": ["teamA"] }, + { "key": "", "role": "admin", "namespaces": ["teamA"] } +] +``` + +Notes: + +- Never reuse the same key for multiple roles unless you must. +- Prefer binding keys to a namespace list instead of `"*"`. + +## Backup strategy (recommended) + +You need to back up three persistent stores: + +- **Neo4j** data volume (`neo4j_data`) +- **Qdrant** storage volume (`qdrant_data`) +- **deep-memory-server data** volume (`deepmem_data`) — includes: + - durable queue directory (`/data/queue`) + - audit log (`/data/audit/audit.jsonl`) + +### Option A: volume tar backups (simple, consistent) + +This is the simplest approach for Docker Compose. It captures raw volumes. + +1. Stop the deep memory stack (to get a consistent snapshot): + +```bash +docker compose --profile deep-memory-prod stop deep-memory-server-prod qdrant neo4j +``` + +2. Create a backup directory: + +```bash +mkdir -p backups/deep-memory +``` + +3. Backup volumes to tarballs: + +```bash +docker run --rm -v neo4j_data:/data -v "$PWD/backups/deep-memory:/backup" alpine \ + sh -lc 'cd /data && tar -czf /backup/neo4j_data.tgz .' + +docker run --rm -v qdrant_data:/data -v "$PWD/backups/deep-memory:/backup" alpine \ + sh -lc 'cd /data && tar -czf /backup/qdrant_data.tgz .' + +docker run --rm -v deepmem_data:/data -v "$PWD/backups/deep-memory:/backup" alpine \ + sh -lc 'cd /data && tar -czf /backup/deepmem_data.tgz .' +``` + +4. Start the stack: + +```bash +docker compose --profile deep-memory-prod up -d +``` + +5. Verify readiness: + +```bash +curl -fsS http://127.0.0.1:8088/readyz +``` + +### Option B: DB-native snapshots (more complex, best for larger deployments) + +If you run at scale, consider: + +- Neo4j: `neo4j-admin database dump` / `load` with an explicit backup directory +- Qdrant: collection snapshots via Qdrant's snapshot API + +This is more operationally involved but can reduce downtime and provide better controls. + +## Restore procedure (Option A tar backups) + +1. Stop the deep memory stack: + +```bash +docker compose --profile deep-memory-prod down +``` + +2. Restore each volume tarball: + +```bash +docker run --rm -v neo4j_data:/data -v "$PWD/backups/deep-memory:/backup" alpine \ + sh -lc 'rm -rf /data/* && cd /data && tar -xzf /backup/neo4j_data.tgz' + +docker run --rm -v qdrant_data:/data -v "$PWD/backups/deep-memory:/backup" alpine \ + sh -lc 'rm -rf /data/* && cd /data && tar -xzf /backup/qdrant_data.tgz' + +docker run --rm -v deepmem_data:/data -v "$PWD/backups/deep-memory:/backup" alpine \ + sh -lc 'rm -rf /data/* && cd /data && tar -xzf /backup/deepmem_data.tgz' +``` + +3. Start the stack: + +```bash +docker compose --profile deep-memory-prod up -d +``` + +4. Verify: + +- `GET /readyz` returns 200 +- a small sample `/retrieve_context` works for a known namespace +- queue stats look sane (`/health`) + +## Disaster recovery drills (recommended) + +Run these drills periodically (for example, monthly) so you can restore quickly under stress. + +### Drill A: backup → restore → verify (tabletop + hands-on) + +1. Pick a **non-production** environment (or a staging clone). +2. Take fresh tar backups of all three volumes: + - `neo4j_data`, `qdrant_data`, `deepmem_data` +3. Restore into a clean environment using the restore procedure above. +4. Verify: + - `GET /readyz` returns 200 + - `GET /health/details` shows schema checks are ok + - `GET /health` shows queue stats are sane + - `POST /retrieve_context` returns 200 for a known namespace + - `POST /update_memory_index` returns 200 (queued or processed) + - If you use async forget: `GET /queue/forget/stats` returns 200 (admin key) + +## Upgrade checklist (deep-memory-server) + +When updating deep-memory-server versions: + +1. Ensure you have a fresh backup (Neo4j/Qdrant/deepmem_data) +2. Confirm API keys still exist and are valid (`API_KEYS_JSON`) +3. `docker compose --profile deep-memory-prod build deep-memory-server-prod` +4. Roll the container: + +```bash +docker compose --profile deep-memory-prod up -d deep-memory-server-prod +``` + +5. Verify: + +- `GET /readyz` (200) +- `GET /health/details` shows schema checks are ok (and no migration warnings) +- `GET /metrics` (authorized admin key) +- normal OpenClaw operation (retrieval + update) +- (recommended) run offline eval regression: `pnpm --dir packages/deep-memory-server eval` + +### Release checklist (deep-memory-server) + +Before shipping a deep-memory-server build to production: + +- Confirm backups exist (Neo4j/Qdrant/deepmem_data) +- Run `pnpm -w check` and `pnpm -w test -F deep-memory-server` +- Run `pnpm --dir packages/deep-memory-server eval` (and compare vs baseline if you keep snapshots) +- Verify `GET /readyz` is 200 on the target environment +- Verify `GET /health/details` as admin: + - schema checks ok + - guardrails settings are as expected (rate limits, backlog, namespace limits, sensitive filter) +- If you set build metadata, verify `/health` includes `build.sha` and `build.time` + +### Schema migrations (Neo4j/Qdrant) + +deep-memory-server performs **startup schema checks** for Neo4j constraints/indexes and Qdrant collection schema. + +Key env vars: + +- `MIGRATIONS_MODE=apply|validate|off` (default `apply`) + - `apply`: create missing safe objects (constraints/indexes/collection) but never drops data + - `validate`: check and report missing/mismatched schema, but do not modify the DB + - `off`: skip schema checks (not recommended) +- `MIGRATIONS_STRICT=true|false` (default `false`) + - If `true`, the server fails startup when schema checks fail (recommended for strict production) + +**Qdrant dims changes:** if you change `VECTOR_DIMS`, the existing collection cannot be resized in-place. +Recommended migration is to create a new collection (new name), run a reindex job, then switch `QDRANT_COLLECTION`. + +### Reindex job (Qdrant collection migration) + +When you need to migrate Qdrant collections (for example, after changing `VECTOR_DIMS` or renaming collections), you can re-embed and repopulate Qdrant from Neo4j: + +```bash +# Reindex a single namespace into a new collection (recommended) +pnpm --dir packages/deep-memory-server reindex -- \ + --namespace teamA \ + --target-collection openclaw_memories_v2 + +# Canary: only process first 200 memories +pnpm --dir packages/deep-memory-server reindex -- \ + --namespace teamA \ + --target-collection openclaw_memories_v2 \ + --max-items 200 + +# Dry-run scan (no embeddings, no Qdrant writes) +pnpm --dir packages/deep-memory-server reindex -- \ + --namespace teamA \ + --target-collection openclaw_memories_v2 \ + --dry-run +``` + +Notes: + +- The reindex job reads `Memory` nodes from Neo4j and writes points to the target Qdrant collection. +- It does **not** delete the old collection. Switch `QDRANT_COLLECTION` after you validate the new one. + +### Validate a reindex (coverage + spot-check) + +After reindexing into a new collection, validate that Qdrant contains the expected points and payloads: + +```bash +pnpm --dir packages/deep-memory-server validate-reindex -- \ + --namespace teamA \ + --target-collection openclaw_memories_v2 \ + --sample-size 50 \ + --out reindex-report.teamA.json +``` + +This tool: + +- Compares **Neo4j count vs Qdrant count** (per namespace) +- Samples memories from Neo4j and verifies the **Qdrant payload exists** and matches expected fields + +### Rollback: Qdrant collection migration + +If a new collection shows degraded recall quality or performance: + +1. Switch deep-memory-server back to the previous `QDRANT_COLLECTION` value. +2. Restart deep-memory-server. +3. Verify: + - `GET /readyz` returns 200 + - `GET /health/details` shows Qdrant schema ok for the reverted collection + - Run `pnpm --dir packages/deep-memory-server eval` (or your real regression suite) + +Keep old collections until you have validated the new one for a full period (at least a few days). + +## Key rotation (API_KEYS_JSON) + +Safe rotation sequence: + +1. Add new keys (keep old keys temporarily) +2. Deploy deep-memory-server with updated `API_KEYS_JSON` +3. Update OpenClaw config to use new keys +4. Remove old keys and redeploy + +## Data lifecycle (expired memories) + +If you store memories with `expires_at`, you should periodically garbage-collect expired items so they do not linger in Qdrant/Neo4j. + +Run an offline GC (recommended during low-traffic windows): + +```bash +# Dry-run first +pnpm --dir packages/deep-memory-server gc-expired -- --namespace teamA --dry-run + +# Then delete expired memories (both Qdrant + Neo4j) +pnpm --dir packages/deep-memory-server gc-expired -- \ + --namespace teamA \ + --delete-batch 200 \ + --out gc-expired.teamA.json +``` + +Notes: + +- Prefer running on a staging restore first if you have not done this before. +- Keep backups before large destructive maintenance. diff --git a/docs/reference/deep-memory-server.md b/docs/reference/deep-memory-server.md new file mode 100644 index 0000000000000..93f08816fcff6 --- /dev/null +++ b/docs/reference/deep-memory-server.md @@ -0,0 +1,176 @@ +--- +summary: "Reference for deep-memory-server configuration, API contract, and guardrail semantics" +read_when: + - You are configuring deep-memory-server (env vars, guardrails, queue) + - You need the API contract (responses, error codes, retry-after semantics) +title: "Deep memory server reference" +--- + +# Deep memory server reference + +This page is the **configuration + API contract** reference for `deep-memory-server`. + +For production operations (backups, restores, upgrades), see [Deep memory ops](/reference/deep-memory-ops). + +## Configuration (environment variables) + +Defaults are from `packages/deep-memory-server/src/config.ts`. + +### Build metadata (optional) + +- `BUILD_SHA`: git SHA (string) +- `BUILD_TIME`: build timestamp ISO string + +These appear in `GET /health` under `build.sha` and `build.time`. + +### Auth / security + +- `REQUIRE_API_KEY` (default `false`) +- `API_KEYS_JSON` (recommended): JSON array of keys with `role` (`read|write|admin`) and `namespaces` allowlist +- Legacy single-role keys: + - `API_KEY` (single key) + - `API_KEYS` (comma-separated keys) + +### Backends + +Qdrant: + +- `QDRANT_URL` (default `http://qdrant:6333`) +- `QDRANT_API_KEY` (optional) +- `QDRANT_COLLECTION` (default `openclaw_memories`) +- `VECTOR_DIMS` (default `384`) + +Neo4j: + +- `NEO4J_URI` (default `bolt://neo4j:7687`) +- `NEO4J_USER` (default `neo4j`) +- `NEO4J_PASSWORD` (default `openclaw`) + +### Queue (durable update + forget) + +- `QUEUE_DIR` (default `./data/queue`) +- `QUEUE_MAX_ATTEMPTS` (default `10`) +- `QUEUE_RETRY_BASE_MS` (default `2000`) +- `QUEUE_RETRY_MAX_MS` (default `300000`) +- `QUEUE_KEEP_DONE` (default `true`) +- `QUEUE_RETENTION_DAYS` (default `7`) +- `QUEUE_MAX_TASK_BYTES` (default `2097152`) +- `UPDATE_CONCURRENCY` (default `1`): worker concurrency for update queue + +### Rate limits (fixed window, per key + route) + +- `RATE_LIMIT_ENABLED` (default `false`) +- `RATE_LIMIT_WINDOW_MS` (default `60000`) +- `RATE_LIMIT_RETRIEVE_PER_WINDOW` (default `0`) +- `RATE_LIMIT_UPDATE_PER_WINDOW` (default `0`) +- `RATE_LIMIT_FORGET_PER_WINDOW` (default `0`) +- `RATE_LIMIT_QUEUE_ADMIN_PER_WINDOW` (default `0`) + +### Update ingestion guardrails (server-side) + +Hard control: + +- `UPDATE_DISABLED_NAMESPACES` (optional): comma/space separated namespace list that is **write-disabled** + +Shaping: + +- `UPDATE_MIN_INTERVAL_MS` (default `0`): minimum spacing between update requests per `namespace::session_id` +- `UPDATE_SAMPLE_RATE` (default `1`): deterministic sampling \(0..1\) for skipping some updates + +Overload handling: + +- `UPDATE_BACKLOG_REJECT_PENDING` (default `0`): if pending >= this, reject async updates with 503 +- `UPDATE_BACKLOG_RETRY_AFTER_SECONDS` (default `30`): `retry-after` for overload responses +- `UPDATE_BACKLOG_READ_ONLY_PENDING` (default `0`): if pending >= this, enter read-only mode for async updates +- `UPDATE_BACKLOG_DELAY_PENDING` (default `0`): if pending >= this, accept but delay queue tasks +- `UPDATE_BACKLOG_DELAY_SECONDS` (default `0`): delay duration when delaying + +Namespace concurrency: + +- `NAMESPACE_UPDATE_CONCURRENCY` (default `0`): max concurrent update tasks per namespace (queue worker) +- `NAMESPACE_RETRIEVE_CONCURRENCY` (default `0`): max concurrent retrieve requests per namespace (API) + +### Retrieve degradation (optional) + +- `RETRIEVE_DEGRADE_RELATED_PENDING` (default `0`): if update queue pending >= this, retrieval degrades by skipping entity/topic hint extraction (reduces Neo4j relation work) + +### Sensitive filtering (write path) + +- `SENSITIVE_FILTER_ENABLED` (default `true`) +- `SENSITIVE_RULESET_VERSION` (default `builtin-v1`): audit-friendly version string +- `SENSITIVE_DENY_REGEX_JSON` (optional): JSON array of regex strings (case-insensitive) to deny +- `SENSITIVE_ALLOW_REGEX_JSON` (optional): JSON array of regex strings (case-insensitive) to allow (overrides deny) + +## API contract (responses + error semantics) + +### Common error semantics + +- **429 rate limiting**: + - body: `{ "error": "rate_limited" }` + - header: `retry-after` +- **503 overload / not ready**: + - body: `{ "error": "", ...details }` + - header: `retry-after` when a retry is appropriate + +### `POST /retrieve_context` + +Success (200): `RetrieveContextResponse` + +Overload (503): + +- `{ "error": "namespace_overloaded", "namespace": "", "active": , "limit": }` + +### `POST /update_memory_index` + +Success (200): + +- processed: `{ "status": "processed", "memories_added": , "memories_filtered": }` + - Optional (only when `async=false` and `return_memory_ids=true`): + - `memory_ids`: string array (up to `max_memory_ids`, default 200) + - `memory_ids_truncated`: boolean (true if the list hit the max) +- queued: `{ "status": "queued", "memories_added": 0, "memories_filtered": 0 }` +- skipped: `{ "status": "skipped", "error": "", "memories_added": 0, "memories_filtered": 0 }` + +Overload (503): + +- `{ "error": "queue_overloaded", "pendingApprox": , "retryAfterSeconds": }` +- `{ "error": "degraded_read_only", "pendingApprox": , "retryAfterSeconds": }` + +### `POST /session/inspect` + +Returns a session-level summary without requiring `user_input`. + +This endpoint is used by higher-level features (for example, RustFS-backed `file_search`) to obtain stable, query-independent semantics (topics/entities) for a session. + +Request body: + +- `namespace` (optional) +- `session_id` (required) +- `limit` (optional, default 100, max 1000) +- `include_content` (optional, default false) + +Success (200): `InspectSessionResponse` (topics/entities aggregation + representative memories; `summary` only when `include_content=true`). + +### `POST /forget` + +Success (200): + +- dry run: `{ "status": "dry_run", "namespace": "", "request_id": "", ... }` +- queued: `{ "status": "queued", "namespace": "", "request_id": "", "task_id": "", "key": "", ... }` +- processed: `{ "status": "processed", "namespace": "", "request_id": "", "deleted": , "results": { ... } }` + +### Queue admin endpoints + +Update queue: + +- `GET /queue/stats` +- `GET /queue/failed` +- `GET /queue/failed/export` +- `POST /queue/failed/retry` + +Forget queue: + +- `GET /queue/forget/stats` +- `GET /queue/forget/failed` +- `GET /queue/forget/failed/export` +- `POST /queue/forget/failed/retry` diff --git a/docs/reference/rustfs-file-tools.md b/docs/reference/rustfs-file-tools.md new file mode 100644 index 0000000000000..acd5ea759062d --- /dev/null +++ b/docs/reference/rustfs-file-tools.md @@ -0,0 +1,59 @@ +--- +summary: "How OpenClaw file tools (RustFS) perform semantic search and safe sending" +read_when: + - You are enabling RustFS + rustfs-worker + - You want semantic file search that is LLM-driven and traceable +title: "RustFS file tools reference" +--- + +# RustFS file tools reference + +This page documents the OpenClaw agent tools backed by **RustFS**: + +- `file_ingest`: upload a workspace file into RustFS +- `file_search`: list/search candidate files (optionally with semantic evidence) +- `file_send`: create a short-lived public link for a chosen file + +## Safety contract (must follow) + +- `file_search` **must not** automatically send files. +- Always ask the user to confirm a specific candidate (by `n` or `fileId`) before calling `file_send`. + +## `file_search` semantic mode (dual evidence) + +When `includeSemantic=true`, `file_search` uses a two-phase approach: + +1. **RustFS coarse search**: list candidates using tenant + metadata filters. +2. **Deep memory evidence** (Top-K only): for each candidate file session `rustfs:file:`, it concurrently calls: + - `POST /retrieve_context` (query-driven evidence): returns `context` and memory matches for the given `user_input` + - `POST /session/inspect` (session-level semantics): returns aggregated `topics/entities` (and optional `summary`) + +This produces a more stable and explainable ranking: + +- **Session semantics** answers “what this file is about”. +- **Query evidence** answers “why it matches this request”. + +## Output fields (candidates) + +`file_search` returns: + +- `candidates[]`: a numbered list intended for user confirmation. +- `clarify` (optional): suggested follow-up questions when disambiguation is needed. + +Each candidate may include: + +- **Basic**: `n`, `fileId`, `filename`, `mime`, `size`, `createdAtMs`, `extractStatus` +- **Ingest hints**: `kind`, `tags`, `hint` (from `openclaw_ingest` and/or `classification`) +- **Semantic evidence** (when enabled): + - `semanticScore`: relevance score used for re-ranking + - `semanticContext`: short context snippet returned by deep-memory retrieval + - `semanticTopics`, `semanticEntities`: session-level aggregated semantics + - `semanticSummary`: optional session summary (if available) + +## Traceability: file_id to memory_ids + +If configured, `rustfs-worker` can request `memory_ids` from `POST /update_memory_index` (sync mode) and store them in: + +- `annotations.deep_memory.memory_ids` + +This lets you trace a RustFS `file_id` to the deep-memory ids that were written/updated for the file session. diff --git a/docs/reference/startup.md b/docs/reference/startup.md new file mode 100644 index 0000000000000..1f2f82ddc3719 --- /dev/null +++ b/docs/reference/startup.md @@ -0,0 +1,110 @@ +--- +summary: "How to start OpenClaw, deep-memory, RustFS, and rustfs-worker (docker-compose profiles)" +read_when: + - You are starting OpenClaw locally or on a server + - You want to enable deep-memory and RustFS file ingestion/search +title: "Startup flow (Docker profiles)" +--- + +# Startup flow (Docker profiles) + +This page describes the **current startup flow** for the OpenClaw stack in this repository, based on `docker-compose.yml`. + +OpenClaw uses **Docker Compose profiles** so you can start components independently (recommended) or together (when needed). + +## Components + +- **Gateway**: `openclaw-gateway` (+ `openclaw-cli` for running commands) +- **Deep memory**: + - dev: `deep-memory-server` + `qdrant` + `neo4j` (profile `deep-memory`) + - prod: `deep-memory-server-prod` + `qdrant` + `neo4j` (profile `deep-memory-prod`) +- **RustFS**: `rustfs` (profile `rustfs`) +- **RustFS worker**: `rustfs-worker` (profile `rustfs-worker`) + +## Compose profiles + +From repo root: + +- Gateway only: + +```bash +docker compose up -d openclaw-gateway +``` + +- Deep memory (dev): + +```bash +docker compose --profile deep-memory up -d +``` + +- Deep memory (prod): + +```bash +docker compose --profile deep-memory-prod up -d +``` + +- RustFS: + +```bash +docker compose --profile rustfs up -d +``` + +- RustFS + deep-memory-prod + worker (recommended for file semantic ingestion): + +```bash +docker compose --profile rustfs --profile deep-memory-prod --profile rustfs-worker up -d +``` + +## Recommended startup order + +1. Start deep-memory (prod) first (brings up Qdrant + Neo4j + deep-memory-server-prod). +2. Start RustFS. +3. Start rustfs-worker (depends on RustFS + deep-memory health checks). +4. Start the gateway. + +This order ensures the worker can immediately index files and the gateway can use the stack. + +## Required environment variables (high-level) + +Gateway + CLI: + +- `OPENCLAW_CONFIG_DIR` and `OPENCLAW_WORKSPACE_DIR` (bind mounts) +- `OPENCLAW_GATEWAY_TOKEN` (control UI token) + +deep-memory-prod (recommended): + +- `OPENCLAW_DEEPMEM_API_KEYS_JSON` (write/read/admin keys + namespaces allowlist) +- `OPENCLAW_NEO4J_PASSWORD` + +RustFS: + +- `OPENCLAW_RUSTFS_API_KEYS_JSON` (API keys bound to tenant) +- `OPENCLAW_RUSTFS_SIGNING_KEY` (required for share links) +- Optional: `OPENCLAW_RUSTFS_MASTER_KEY` (encryption at rest) + +rustfs-worker: + +- `OPENCLAW_RUSTFS_WORKER_API_KEY` (RustFS key with permission to claim/download/annotate) +- `OPENCLAW_DEEPMEM_WORKER_API_KEY` (deep-memory write key) +- Optional: `OPENCLAW_DEEPMEM_NAMESPACE` + +Note: use a `.env` file or your host environment to provide these values; never commit secrets. + +## Verification checklist + +- deep-memory-prod: + - `GET http://127.0.0.1:${OPENCLAW_DEEPMEM_PORT:-8088}/readyz` +- RustFS: + - `GET http://127.0.0.1:${OPENCLAW_RUSTFS_PORT:-8099}/readyz` +- Gateway: + - Open `http://127.0.0.1:${OPENCLAW_GATEWAY_PORT:-18789}/` and paste the token in Settings. + +## Troubleshooting (common) + +- Compose profiles didn’t start anything: + - Confirm you passed the right `--profile ...` flags. +- Worker starts but does nothing: + - Ensure files were ingested into RustFS and `extract_status` is `pending`. + - Check `rustfs-worker` logs for overload/backoff or auth failures. +- deep-memory overload responses: + - `503 queue_overloaded` or `503 degraded_read_only` indicates backlog. Increase capacity or reduce ingestion rate. diff --git a/docs/tools/slash-commands.md b/docs/tools/slash-commands.md index dea4fb0d30f1b..05aaa16d2196e 100644 --- a/docs/tools/slash-commands.md +++ b/docs/tools/slash-commands.md @@ -75,6 +75,8 @@ Text + native (when enabled): - `/status` (show current status; includes provider usage/quota for the current model provider when available) - `/allowlist` (list/add/remove allowlist entries) - `/approve allow-once|allow-always|deny` (resolve exec approval prompts) +- `/forget [confirm]` (delete deep memory for the current session; **admin-only**; defaults to dry-run unless `confirm` is provided) +- `/deepmemory status [details] [queue]` (deep memory server health/readiness/queue; **admin-only**; alias: `/deep-memory`) - `/context [list|detail|json]` (explain “context”; `detail` shows per-file + per-tool + per-skill + system prompt size) - `/export-session [path]` (alias: `/export`) (export current session to HTML with full system prompt) - `/whoami` (show your sender id; alias: `/id`) diff --git a/docs/zh-CN/reference/startup.md b/docs/zh-CN/reference/startup.md new file mode 100644 index 0000000000000..bf62de5c64184 --- /dev/null +++ b/docs/zh-CN/reference/startup.md @@ -0,0 +1,111 @@ +--- +summary: "OpenClaw + deep-memory + RustFS + rustfs-worker 的 Docker 启动流程(profiles 组合)" +read_when: + - 你在本机或服务器上启动 OpenClaw + - 你要启用 deep-memory 与 RustFS 文件归档/语义检索 +title: "启动流程(Docker profiles)" +--- + +# 启动流程(Docker profiles) + +本文梳理本仓库当前的启动方式:基于 `docker-compose.yml`,通过 **Docker Compose profiles** 将网关、deep-memory、RustFS、worker 进行**可选分组启动**。 + +## 组件一览 + +- **Gateway(网关)**:`openclaw-gateway`(配套 `openclaw-cli` 用于执行命令) +- **deep-memory(语义记忆)** + - 开发模式:`deep-memory-server` + `qdrant` + `neo4j`(profile:`deep-memory`) + - 生产模式:`deep-memory-server-prod` + `qdrant` + `neo4j`(profile:`deep-memory-prod`) +- **RustFS(文件归档)**:`rustfs`(profile:`rustfs`) +- **rustfs-worker(抽取/入库后台任务)**:`rustfs-worker`(profile:`rustfs-worker`) + +## 关键点:默认是分开启动 + +除 `openclaw-gateway/openclaw-cli` 外,其余组件都挂在 profile 上,意味着: + +- 你不显式指定 `--profile ...` 时,它们不会启动 +- 你可以按需组合多个 profile,达到“一起启动”的效果 + +## 常用启动命令(从仓库根目录执行) + +- 仅启动网关: + +```bash +docker compose up -d openclaw-gateway +``` + +- 启动 deep-memory(开发模式): + +```bash +docker compose --profile deep-memory up -d +``` + +- 启动 deep-memory(生产模式,推荐): + +```bash +docker compose --profile deep-memory-prod up -d +``` + +- 启动 RustFS: + +```bash +docker compose --profile rustfs up -d +``` + +- 启动 RustFS + deep-memory-prod + rustfs-worker(推荐的“文件语义入库”组合): + +```bash +docker compose --profile rustfs --profile deep-memory-prod --profile rustfs-worker up -d +``` + +## 推荐启动顺序(稳态) + +1. 先启动 `deep-memory-prod`(会带起 Qdrant + Neo4j + deep-memory-server-prod) +2. 再启动 `rustfs` +3. 再启动 `rustfs-worker`(依赖 rustfs 与 deep-memory 的 healthcheck) +4. 最后启动 `openclaw-gateway` + +这样 worker 可以立刻 claim 待抽取文件并入库,网关侧也能稳定使用相关能力。 + +## 环境变量(高层必需项) + +网关/CLI: + +- `OPENCLAW_CONFIG_DIR`、`OPENCLAW_WORKSPACE_DIR`(绑定到容器内 `~/.openclaw` 与 workspace) +- `OPENCLAW_GATEWAY_TOKEN`(控制台 UI token) + +deep-memory-prod(推荐): + +- `OPENCLAW_DEEPMEM_API_KEYS_JSON`(建议用 JSON 定义 read/write/admin key,并配置 namespaces allowlist) +- `OPENCLAW_NEO4J_PASSWORD` + +RustFS: + +- `OPENCLAW_RUSTFS_API_KEYS_JSON`(API key 绑定 tenant) +- `OPENCLAW_RUSTFS_SIGNING_KEY`(**必需**:用于文件分享链接签名) +- 可选:`OPENCLAW_RUSTFS_MASTER_KEY`(开启新文件静态加密) + +rustfs-worker: + +- `OPENCLAW_RUSTFS_WORKER_API_KEY`(RustFS 权限:claim/download/annotations/status) +- `OPENCLAW_DEEPMEM_WORKER_API_KEY`(deep-memory 写权限) +- 可选:`OPENCLAW_DEEPMEM_NAMESPACE` + +建议通过 `.env` 或宿主机环境变量注入;不要把任何真实 key 提交到 git。 + +## 启动后怎么确认“真的跑起来了” + +- deep-memory-prod: + - `GET http://127.0.0.1:${OPENCLAW_DEEPMEM_PORT:-8088}/readyz` +- RustFS: + - `GET http://127.0.0.1:${OPENCLAW_RUSTFS_PORT:-8099}/readyz` +- Gateway: + - 打开 `http://127.0.0.1:${OPENCLAW_GATEWAY_PORT:-18789}/`,在 Settings 粘贴 token + +## 常见问题 + +- profile 没带导致“怎么没启动”: + - 记得加 `--profile deep-memory-prod` / `--profile rustfs` / `--profile rustfs-worker` +- worker 一直空转: + - 先确认 RustFS 里有文件、且 `extract_status` 为 `pending`/`processing(过期)` 可 claim + - 查看 worker 日志是否 auth 失败、或 deep-memory 503 过载退避 diff --git a/extensions/feishu/src/bot.ts b/extensions/feishu/src/bot.ts index 943ed964b51c7..c2ce82a35c7f3 100644 --- a/extensions/feishu/src/bot.ts +++ b/extensions/feishu/src/bot.ts @@ -15,6 +15,7 @@ import { resolveFeishuAccount } from "./accounts.js"; import { createFeishuClient } from "./client.js"; import { tryRecordMessagePersistent } from "./dedup.js"; import { maybeCreateDynamicAgent } from "./dynamic-agent.js"; +import { extractPermissionError, type FeishuPermissionErrorInfo } from "./errors.js"; import { normalizeFeishuExternalKey } from "./external-keys.js"; import { downloadMessageResourceFeishu } from "./media.js"; import { @@ -35,42 +36,7 @@ import { getMessageFeishu, sendMessageFeishu } from "./send.js"; import type { FeishuMessageContext, FeishuMediaInfo, ResolvedFeishuAccount } from "./types.js"; import type { DynamicAgentCreationConfig } from "./types.js"; -// --- Permission error extraction --- -// Extract permission grant URL from Feishu API error response. -type PermissionError = { - code: number; - message: string; - grantUrl?: string; -}; - -function extractPermissionError(err: unknown): PermissionError | null { - if (!err || typeof err !== "object") return null; - - // Axios error structure: err.response.data contains the Feishu error - const axiosErr = err as { response?: { data?: unknown } }; - const data = axiosErr.response?.data; - if (!data || typeof data !== "object") return null; - - const feishuErr = data as { - code?: number; - msg?: string; - error?: { permission_violations?: Array<{ uri?: string }> }; - }; - - // Feishu permission error code: 99991672 - if (feishuErr.code !== 99991672) return null; - - // Extract the grant URL from the error message (contains the direct link) - const msg = feishuErr.msg ?? ""; - const urlMatch = msg.match(/https:\/\/[^\s,]+\/app\/[^\s,]+/); - const grantUrl = urlMatch?.[0]; - - return { - code: feishuErr.code, - message: msg, - grantUrl, - }; -} +type PermissionError = FeishuPermissionErrorInfo; // --- Sender name resolution (so the agent can distinguish who is speaking in group chats) --- // Cache display names by open_id to avoid an API call on every message. @@ -341,23 +307,24 @@ async function resolveFeishuMediaList(params: { maxBytes: number; log?: (msg: string) => void; accountId?: string; -}): Promise { +}): Promise<{ media: FeishuMediaInfo[]; permissionError?: PermissionError }> { const { cfg, messageId, messageType, content, maxBytes, log, accountId } = params; // Only process media message types (including post for embedded images) const mediaTypes = ["image", "file", "audio", "video", "sticker", "post"]; if (!mediaTypes.includes(messageType)) { - return []; + return { media: [] }; } const out: FeishuMediaInfo[] = []; + let permissionError: PermissionError | undefined; const core = getFeishuRuntime(); // Handle post (rich text) messages with embedded images if (messageType === "post") { const { imageKeys } = parsePostContent(content); if (imageKeys.length === 0) { - return []; + return { media: [] }; } log?.(`feishu: post message contains ${imageKeys.length} embedded image(s)`); @@ -393,17 +360,21 @@ async function resolveFeishuMediaList(params: { log?.(`feishu: downloaded embedded image ${imageKey}, saved to ${saved.path}`); } catch (err) { + const permErr = extractPermissionError(err); + if (permErr && !permissionError) { + permissionError = permErr; + } log?.(`feishu: failed to download embedded image ${imageKey}: ${String(err)}`); } } - return out; + return { media: out, permissionError }; } // Handle other media types const mediaKeys = parseMediaKeys(content, messageType); if (!mediaKeys.imageKey && !mediaKeys.fileKey) { - return []; + return { media: [] }; } try { @@ -415,7 +386,7 @@ async function resolveFeishuMediaList(params: { // The image.get API is only for images uploaded via im/v1/images, not for message attachments const fileKey = mediaKeys.fileKey || mediaKeys.imageKey; if (!fileKey) { - return []; + return { media: [] }; } const resourceType = messageType === "image" ? "image" : "file"; @@ -452,10 +423,14 @@ async function resolveFeishuMediaList(params: { log?.(`feishu: downloaded ${messageType} media, saved to ${saved.path}`); } catch (err) { + const permErr = extractPermissionError(err); + if (permErr) { + permissionError = permErr; + } log?.(`feishu: failed to download ${messageType} media: ${String(err)}`); } - return out; + return { media: out, permissionError }; } /** @@ -852,7 +827,7 @@ export async function handleFeishuMessage(params: { // Resolve media from message const mediaMaxBytes = (feishuCfg?.mediaMaxMb ?? 30) * 1024 * 1024; // 30MB default - const mediaList = await resolveFeishuMediaList({ + const mediaResult = await resolveFeishuMediaList({ cfg, messageId: ctx.messageId, messageType: event.message.message_type, @@ -861,6 +836,10 @@ export async function handleFeishuMessage(params: { log, accountId: account.accountId, }); + const mediaList = mediaResult.media; + if (mediaResult.permissionError && !permissionErrorForAgent) { + permissionErrorForAgent = mediaResult.permissionError; + } const mediaPayload = buildAgentMediaPayload(mediaList); // Fetch quoted/replied message content if parentId exists diff --git a/extensions/feishu/src/doc-schema.ts b/extensions/feishu/src/doc-schema.ts index d92173c0ff5d6..3c0ff23cfc457 100644 --- a/extensions/feishu/src/doc-schema.ts +++ b/extensions/feishu/src/doc-schema.ts @@ -5,6 +5,20 @@ export const FeishuDocSchema = Type.Union([ action: Type.Literal("read"), doc_token: Type.String({ description: "Document token (extract from URL /docx/XXX)" }), }), + Type.Object({ + action: Type.Literal("export"), + doc_token: Type.String({ + description: "Document token (extract from URL /docx/XXX or from file/drive)", + }), + target_dir: Type.Optional( + Type.String({ description: "Target directory for saving (default: workspace memory dir)" }), + ), + extract_to_memory: Type.Optional( + Type.Boolean({ + description: "Also trigger memory extraction for deep memory (default: true)", + }), + ), + }), Type.Object({ action: Type.Literal("write"), doc_token: Type.String({ description: "Document token" }), diff --git a/extensions/feishu/src/docx.ts b/extensions/feishu/src/docx.ts index a934f1d3aa895..ea96b853ae2eb 100644 --- a/extensions/feishu/src/docx.ts +++ b/extensions/feishu/src/docx.ts @@ -1,11 +1,12 @@ import { promises as fs } from "node:fs"; -import { basename } from "node:path"; +import path from "node:path"; import { Readable } from "stream"; import type * as Lark from "@larksuiteoapi/node-sdk"; import { Type } from "@sinclair/typebox"; import type { OpenClawPluginApi } from "openclaw/plugin-sdk"; import { listEnabledFeishuAccounts } from "./accounts.js"; import { FeishuDocSchema, type FeishuDocParams } from "./doc-schema.js"; +import { FeishuApiError, extractPermissionError } from "./errors.js"; import { getFeishuRuntime } from "./runtime.js"; import { createFeishuToolClient, @@ -219,7 +220,7 @@ async function resolveUploadInput( } return { buffer, - fileName: explicitFileName || basename(filePath!), + fileName: explicitFileName || path.basename(filePath!), }; } @@ -405,7 +406,22 @@ async function readDoc(client: Lark.Client, docToken: string) { ]); if (contentRes.code !== 0) { - throw new Error(contentRes.msg); + throw new FeishuApiError({ + code: contentRes.code ?? -1, + message: contentRes.msg ?? "Feishu doc rawContent failed", + }); + } + if (infoRes.code !== 0) { + throw new FeishuApiError({ + code: infoRes.code ?? -1, + message: infoRes.msg ?? "Feishu doc get failed", + }); + } + if (blocksRes.code !== 0) { + throw new FeishuApiError({ + code: blocksRes.code ?? -1, + message: blocksRes.msg ?? "Feishu doc blocks list failed", + }); } const blocks = blocksRes.data?.items ?? []; @@ -772,6 +788,81 @@ async function deleteBlock(client: Lark.Client, docToken: string, blockId: strin return { success: true, deleted_block_id: blockId }; } +/** + * Export Feishu document content and optionally save to file. + * + * We use docx rawContent here to avoid SDK drift around drive export helpers. + */ +async function exportDoc( + client: Lark.Client, + docToken: string, + targetDir?: string, + extractToMemory: boolean = true, +) { + // Get document info first + const infoRes = await client.docx.document.get({ + path: { document_id: docToken }, + }); + if (infoRes.code !== 0) { + throw new FeishuApiError({ + code: infoRes.code ?? -1, + message: infoRes.msg ?? "Feishu doc get failed", + }); + } + + const docTitle = infoRes.data?.document?.title || `feishu_doc_${docToken}`; + const fileName = `${docTitle.replace(/[^a-zA-Z0-9\u4e00-\u9fa5-_]/g, "_")}.md`; + + const contentRes = await client.docx.document.rawContent({ path: { document_id: docToken } }); + if (contentRes.code !== 0) { + throw new FeishuApiError({ + code: contentRes.code ?? -1, + message: contentRes.msg ?? "Feishu doc rawContent failed", + }); + } + + const content = contentRes.data?.content ?? ""; + + // Build result + const result: { + doc_token: string; + title: string; + file_name: string; + content_length: number; + content_preview: string; + extracted_to_memory: boolean; + saved_to?: string; + file_size?: number; + file_save_error?: string; + } = { + doc_token: docToken, + title: docTitle, + file_name: fileName, + content_length: content.length, + content_preview: content.substring(0, 500) + (content.length > 500 ? "..." : ""), + extracted_to_memory: extractToMemory, + }; + + // If target directory specified, save to file + if (targetDir) { + try { + // Ensure target directory exists + await fs.mkdir(targetDir, { recursive: true }); + + const filePath = path.join(targetDir, fileName); + await fs.writeFile(filePath, content, "utf-8"); + + result.saved_to = filePath; + result.file_size = content.length; + } catch (err) { + const error = err instanceof Error ? err.message : String(err); + result.file_save_error = error; + } + } + + return result; +} + async function listBlocks(client: Lark.Client, docToken: string) { const res = await client.docx.documentBlock.list({ path: { document_id: docToken }, @@ -823,7 +914,6 @@ export function registerFeishuDocTools(api: OpenClawPluginApi) { return; } - // Check if any account is configured const accounts = listEnabledFeishuAccounts(api.config); if (accounts.length === 0) { api.logger.debug?.("feishu_doc: No Feishu accounts configured, skipping doc tools"); @@ -853,11 +943,14 @@ export function registerFeishuDocTools(api: OpenClawPluginApi) { api.registerTool( (ctx) => { const defaultAccountId = ctx.agentAccountId; + const defaultTargetDir = ctx.workspaceDir + ? path.join(ctx.workspaceDir, "downloads", "feishu") + : undefined; return { name: "feishu_doc", label: "Feishu Doc", description: - "Feishu document operations. Actions: read, write, append, create, list_blocks, get_block, update_block, delete_block, create_table, write_table_cells, create_table_with_values, upload_image, upload_file", + "Feishu document operations. Actions: read, write, append, create, export (saves to workspace for RustFS ingest), list_blocks, get_block, update_block, delete_block, create_table, write_table_cells, create_table_with_values, upload_image, upload_file. On permission errors, returns a grant URL when available.", parameters: FeishuDocSchema, async execute(_toolCallId, params) { const p = params as FeishuDocExecuteParams; @@ -866,6 +959,15 @@ export function registerFeishuDocTools(api: OpenClawPluginApi) { switch (p.action) { case "read": return json(await readDoc(client, p.doc_token)); + case "export": + return json( + await exportDoc( + client, + p.doc_token, + p.target_dir ?? defaultTargetDir, + p.extract_to_memory ?? true, + ), + ); case "write": return json( await writeDoc( @@ -959,6 +1061,10 @@ export function registerFeishuDocTools(api: OpenClawPluginApi) { return json({ error: `Unknown action: ${(p as any).action}` }); } } catch (err) { + const permErr = extractPermissionError(err); + if (permErr) { + return json({ error: permErr.message, permission: permErr }); + } return json({ error: err instanceof Error ? err.message : String(err) }); } }, diff --git a/extensions/feishu/src/drive.ts b/extensions/feishu/src/drive.ts index d4bde43aff395..c2185dc29fb74 100644 --- a/extensions/feishu/src/drive.ts +++ b/extensions/feishu/src/drive.ts @@ -2,6 +2,7 @@ import type * as Lark from "@larksuiteoapi/node-sdk"; import type { OpenClawPluginApi } from "openclaw/plugin-sdk"; import { listEnabledFeishuAccounts } from "./accounts.js"; import { FeishuDriveSchema, type FeishuDriveParams } from "./drive-schema.js"; +import { extractPermissionError } from "./errors.js"; import { createFeishuToolClient, resolveAnyEnabledFeishuToolsConfig } from "./tool-account.js"; // ============ Helpers ============ @@ -194,7 +195,7 @@ export function registerFeishuDriveTools(api: OpenClawPluginApi) { name: "feishu_drive", label: "Feishu Drive", description: - "Feishu cloud storage operations. Actions: list, info, create_folder, move, delete", + "Feishu cloud storage operations. Actions: list, info, create_folder, move, delete. On permission errors, returns a grant URL when available.", parameters: FeishuDriveSchema, async execute(_toolCallId, params) { const p = params as FeishuDriveExecuteParams; @@ -220,6 +221,10 @@ export function registerFeishuDriveTools(api: OpenClawPluginApi) { return json({ error: `Unknown action: ${(p as any).action}` }); } } catch (err) { + const permErr = extractPermissionError(err); + if (permErr) { + return json({ error: permErr.message, permission: permErr }); + } return json({ error: err instanceof Error ? err.message : String(err) }); } }, @@ -227,6 +232,4 @@ export function registerFeishuDriveTools(api: OpenClawPluginApi) { }, { name: "feishu_drive" }, ); - - api.logger.info?.(`feishu_drive: Registered feishu_drive tool`); } diff --git a/extensions/feishu/src/errors.ts b/extensions/feishu/src/errors.ts new file mode 100644 index 0000000000000..e81b456355406 --- /dev/null +++ b/extensions/feishu/src/errors.ts @@ -0,0 +1,66 @@ +export const FEISHU_PERMISSION_ERROR_CODE = 99991672; + +export type FeishuPermissionErrorInfo = { + code: number; + message: string; + grantUrl?: string; +}; + +function extractGrantUrlFromMessage(msg: string): string | undefined { + // Feishu often embeds a permission grant URL directly in the error message. + // Example format contains "... https://open.feishu.cn/app/...". + const match = msg.match(/https:\/\/[^\s,]+\/app\/[^\s,]+/); + return match?.[0]; +} + +export class FeishuApiError extends Error { + readonly code: number; + readonly grantUrl?: string; + + constructor(params: { code: number; message: string }) { + super(params.message); + this.name = "FeishuApiError"; + this.code = params.code; + this.grantUrl = + params.code === FEISHU_PERMISSION_ERROR_CODE + ? extractGrantUrlFromMessage(params.message) + : undefined; + } +} + +export function isFeishuApiError(err: unknown): err is FeishuApiError { + return err instanceof FeishuApiError; +} + +/** + * Best-effort extraction of Feishu permission error info from: + * - our FeishuApiError wrapper + * - SDK/HTTP client errors that include `response.data.{code,msg}` + */ +export function extractPermissionError(err: unknown): FeishuPermissionErrorInfo | null { + if (!err || typeof err !== "object") { + return null; + } + + if (err instanceof FeishuApiError && err.code === FEISHU_PERMISSION_ERROR_CODE) { + return { code: err.code, message: err.message, grantUrl: err.grantUrl }; + } + + // Axios-like error: err.response.data contains the Feishu error. + const axiosErr = err as { response?: { data?: unknown } }; + const data = axiosErr.response?.data; + if (!data || typeof data !== "object") { + return null; + } + + const feishuErr = data as { code?: number; msg?: string }; + if (feishuErr.code !== FEISHU_PERMISSION_ERROR_CODE) { + return null; + } + const msg = feishuErr.msg ?? ""; + return { + code: feishuErr.code, + message: msg, + grantUrl: extractGrantUrlFromMessage(msg), + }; +} diff --git a/extensions/feishu/src/media.ts b/extensions/feishu/src/media.ts index 62dfd9551c886..3a65c136d2b96 100644 --- a/extensions/feishu/src/media.ts +++ b/extensions/feishu/src/media.ts @@ -4,6 +4,7 @@ import { Readable } from "stream"; import { withTempDownloadPath, type ClawdbotConfig } from "openclaw/plugin-sdk"; import { resolveFeishuAccount } from "./accounts.js"; import { createFeishuClient } from "./client.js"; +import { FeishuApiError } from "./errors.js"; import { normalizeFeishuExternalKey } from "./external-keys.js"; import { getFeishuRuntime } from "./runtime.js"; import { assertFeishuMessageApiSuccess, toFeishuSendResult } from "./send-result.js"; @@ -29,7 +30,10 @@ async function readFeishuResponseBuffer(params: { // eslint-disable-next-line @typescript-eslint/no-explicit-any -- SDK response type const responseAny = response as any; if (responseAny.code !== undefined && responseAny.code !== 0) { - throw new Error(`${params.errorPrefix}: ${responseAny.msg || `code ${responseAny.code}`}`); + throw new FeishuApiError({ + code: Number(responseAny.code), + message: `${params.errorPrefix}: ${responseAny.msg || `code ${responseAny.code}`}`, + }); } if (Buffer.isBuffer(response)) { @@ -102,7 +106,6 @@ export async function downloadImageFeishu(params: { const response = await client.im.image.get({ path: { image_key: normalizedImageKey }, }); - const buffer = await readFeishuResponseBuffer({ response, tmpDirPrefix: "openclaw-feishu-img-", @@ -138,7 +141,6 @@ export async function downloadMessageResourceFeishu(params: { path: { message_id: messageId, file_key: normalizedFileKey }, params: { type }, }); - const buffer = await readFeishuResponseBuffer({ response, tmpDirPrefix: "openclaw-feishu-resource-", diff --git a/extensions/feishu/src/perm.ts b/extensions/feishu/src/perm.ts index 92c3bb8cdd937..9fece549f0c10 100644 --- a/extensions/feishu/src/perm.ts +++ b/extensions/feishu/src/perm.ts @@ -1,6 +1,7 @@ import type * as Lark from "@larksuiteoapi/node-sdk"; import type { OpenClawPluginApi } from "openclaw/plugin-sdk"; import { listEnabledFeishuAccounts } from "./accounts.js"; +import { extractPermissionError } from "./errors.js"; import { FeishuPermSchema, type FeishuPermParams } from "./perm-schema.js"; import { createFeishuToolClient, resolveAnyEnabledFeishuToolsConfig } from "./tool-account.js"; @@ -168,6 +169,10 @@ export function registerFeishuPermTools(api: OpenClawPluginApi) { return json({ error: `Unknown action: ${(p as any).action}` }); } } catch (err) { + const permErr = extractPermissionError(err); + if (permErr) { + return json({ error: permErr.message, permission: permErr }); + } return json({ error: err instanceof Error ? err.message : String(err) }); } }, @@ -175,6 +180,4 @@ export function registerFeishuPermTools(api: OpenClawPluginApi) { }, { name: "feishu_perm" }, ); - - api.logger.info?.(`feishu_perm: Registered feishu_perm tool`); } diff --git a/extensions/feishu/src/wiki.ts b/extensions/feishu/src/wiki.ts index 0c4383b064765..b31cbccc36080 100644 --- a/extensions/feishu/src/wiki.ts +++ b/extensions/feishu/src/wiki.ts @@ -1,6 +1,7 @@ import type * as Lark from "@larksuiteoapi/node-sdk"; import type { OpenClawPluginApi } from "openclaw/plugin-sdk"; import { listEnabledFeishuAccounts } from "./accounts.js"; +import { extractPermissionError } from "./errors.js"; import { createFeishuToolClient, resolveAnyEnabledFeishuToolsConfig } from "./tool-account.js"; import { FeishuWikiSchema, type FeishuWikiParams } from "./wiki-schema.js"; @@ -182,7 +183,7 @@ export function registerFeishuWikiTools(api: OpenClawPluginApi) { name: "feishu_wiki", label: "Feishu Wiki", description: - "Feishu knowledge base operations. Actions: spaces, nodes, get, create, move, rename", + "Feishu knowledge base operations. Actions: spaces, nodes, get, create, move, rename. On permission errors, returns a grant URL when available.", parameters: FeishuWikiSchema, async execute(_toolCallId, params) { const p = params as FeishuWikiExecuteParams; @@ -225,6 +226,10 @@ export function registerFeishuWikiTools(api: OpenClawPluginApi) { return json({ error: `Unknown action: ${(p as any).action}` }); } } catch (err) { + const permErr = extractPermissionError(err); + if (permErr) { + return json({ error: permErr.message, permission: permErr }); + } return json({ error: err instanceof Error ? err.message : String(err) }); } }, @@ -232,6 +237,4 @@ export function registerFeishuWikiTools(api: OpenClawPluginApi) { }, { name: "feishu_wiki" }, ); - - api.logger.info?.(`feishu_wiki: Registered feishu_wiki tool`); } diff --git a/packages/deep-memory-server/Dockerfile b/packages/deep-memory-server/Dockerfile new file mode 100644 index 0000000000000..367dfecad7381 --- /dev/null +++ b/packages/deep-memory-server/Dockerfile @@ -0,0 +1,41 @@ +FROM node:22-slim AS build + +WORKDIR /repo + +# pnpm via corepack (matches repo workflow) +RUN corepack enable + +# Some deps may compile during install. Also, onnxruntime-node requires glibc at runtime, +# so we build + run on Debian (slim) instead of Alpine. +RUN apt-get update \ + && apt-get install -y --no-install-recommends git python3 make g++ cmake ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Workspace skeleton (keeps install cacheable) +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ +COPY tsconfig.json ./tsconfig.json + +COPY packages/deep-memory-server/package.json packages/deep-memory-server/package.json + +# Install only this package (do NOT use --workspace-root/-w, it forces installing root deps) +RUN pnpm install --frozen-lockfile --filter @openclaw/deep-memory-server + +# Build +COPY packages/deep-memory-server/ packages/deep-memory-server/ +RUN pnpm --dir packages/deep-memory-server build + +# Create a production deploy directory with only prod deps + dist +# pnpm v10 requires injected workspace packages unless using legacy deploy. +RUN pnpm --filter @openclaw/deep-memory-server deploy --prod --legacy /out + +FROM node:22-slim AS runtime + +ENV NODE_ENV=production +WORKDIR /app + +COPY --from=build /out/ /app/ + +EXPOSE 8088 + +CMD ["node", "dist/index.js"] + diff --git a/packages/deep-memory-server/README.md b/packages/deep-memory-server/README.md new file mode 100644 index 0000000000000..aa15bf372eae3 --- /dev/null +++ b/packages/deep-memory-server/README.md @@ -0,0 +1,73 @@ +# Deep Memory Server (OpenClaw) + +This is an **optional** companion service that implements “deep memory” as an external long-term memory backend. + +It exposes two HTTP endpoints used by OpenClaw: + +- `POST /retrieve_context` — retrieve cross-session memory and return a formatted context block. +- `POST /update_memory_index` — ingest session transcript messages and update the long-term memory index. + +## Run (Docker Compose) + +The repo root `docker-compose.yml` includes an optional profile: + +```bash +docker compose --profile deep-memory up +``` + +For production-oriented usage (built image + persistent queue/audit volumes), use: + +```bash +docker compose --profile deep-memory-prod up -d +``` + +Then point OpenClaw at the service: + +```json5 +agents: { + defaults: { + deepMemory: { + enabled: true, + baseUrl: "http://127.0.0.1:8088" + } + } +} +``` + +## Endpoints + +- `GET /health`: liveness + queue/config guardrail summary +- `GET /health/details`: detailed health (includes dependency address summary; admin-only when API keys are required) +- `GET /readyz`: readiness (Qdrant + Neo4j + queue stats; returns 503 on dependency failure) +- `GET /metrics`: Prometheus metrics (admin-only when API keys are required) + +## Environment + +See `src/config.ts` for the full env list and defaults (Neo4j/Qdrant URLs, collection name, vector dims, etc.). + +Notable knobs: + +- `IMPORTANCE_THRESHOLD`: minimum importance to store a memory +- `DEDUPE_SCORE`: Qdrant similarity above this reuses the existing memory id +- `DECAY_HALF_LIFE_DAYS`: time decay half-life applied during retrieval ranking +- `IMPORTANCE_BOOST` / `FREQUENCY_BOOST`: ranking boosts to model “memory growth” +- `RELATED_TOPK`: build `RELATED_TO` graph edges by linking each new memory to its nearest neighbors +- `SENSITIVE_FILTER_ENABLED`: drop likely-secret text (tokens/passwords/keys) before storage +- `MIGRATIONS_MODE`: schema migration behavior (`off` | `validate` | `apply`); default `apply` +- `MIGRATIONS_STRICT`: if `true`, fail startup when schema checks fail (recommended for strict production) + +## Production notes + +- Strongly recommended: set `REQUIRE_API_KEY=true` and configure `API_KEYS_JSON` with per-role keys (`read`, `write`, `admin`). +- Keep `QUEUE_DIR` on persistent storage (container volume) so async ingestion survives restarts. +- Use `/metrics` + `/readyz` for monitoring and alerting (queue backlog, error rates, dependency availability). + +## Reindex (Qdrant collection migration) + +If you need to repopulate a Qdrant collection (for example after changing `VECTOR_DIMS`), you can re-embed memories from Neo4j: + +```bash +pnpm --dir packages/deep-memory-server reindex -- \ + --namespace default \ + --target-collection openclaw_memories_v2 +``` diff --git a/packages/deep-memory-server/eval/dataset.sample.json b/packages/deep-memory-server/eval/dataset.sample.json new file mode 100644 index 0000000000000..5f937048c3431 --- /dev/null +++ b/packages/deep-memory-server/eval/dataset.sample.json @@ -0,0 +1,255 @@ +{ + "version": 1, + "cases": [ + { + "name": "filters expired ephemeral memory and resolves conflict by memory_key", + "namespace": "teamA", + "now": "2026-02-01T00:00:00.000Z", + "query": { + "userInput": "What's my timezone preference?", + "sessionId": "sess-1", + "entities": ["timezone"], + "topics": ["preferences"], + "maxMemories": 3 + }, + "qdrantHits": [ + { + "id": "teamA::mem_tz_old", + "score": 0.91, + "payload": { + "content": "User prefers timezone: UTC.", + "importance": 0.8, + "frequency": 2, + "created_at": "2025-12-01T00:00:00.000Z", + "updated_at": "2026-01-15T00:00:00.000Z", + "kind": "preference", + "memory_key": "preference:timezone", + "subject": "timezone" + } + }, + { + "id": "teamA::mem_tz_new", + "score": 0.88, + "payload": { + "content": "User prefers timezone: Asia/Shanghai.", + "importance": 0.9, + "frequency": 1, + "created_at": "2026-01-20T00:00:00.000Z", + "updated_at": "2026-01-25T00:00:00.000Z", + "kind": "preference", + "memory_key": "preference:timezone", + "subject": "timezone" + } + }, + { + "id": "teamA::mem_ephemeral", + "score": 0.95, + "payload": { + "content": "Temporary code: 123456 (expires soon).", + "importance": 0.7, + "frequency": 1, + "created_at": "2026-01-31T00:00:00.000Z", + "kind": "ephemeral", + "expires_at": "2026-01-31T12:00:00.000Z" + } + } + ], + "neo4jHits": [ + { + "id": "teamA::mem_tz_new", + "content": "User prefers timezone: Asia/Shanghai.", + "importance": 0.9, + "frequency": 1, + "lastSeenAt": "2026-01-25T00:00:00.000Z", + "relationScore": 0.9, + "kind": "preference", + "memoryKey": "preference:timezone", + "subject": "timezone" + } + ], + "expect": { + "includeIds": ["teamA::mem_tz_new"], + "excludeIds": ["teamA::mem_ephemeral"], + "top1Id": "teamA::mem_tz_new", + "uniqueByMemoryKey": true + } + }, + { + "name": "semantic only: returns top semantic hit", + "namespace": "teamA", + "now": "2026-02-01T00:00:00.000Z", + "query": { + "userInput": "What project am I working on?", + "sessionId": "sess-2", + "entities": ["project"], + "topics": ["work"], + "maxMemories": 2 + }, + "qdrantHits": [ + { + "id": "teamA::mem_proj", + "score": 0.9, + "payload": { + "content": "User is working on Project Atlas.", + "importance": 0.7, + "frequency": 1, + "created_at": "2026-01-01T00:00:00.000Z", + "updated_at": "2026-01-20T00:00:00.000Z", + "kind": "fact", + "memory_key": "fact:project" + } + } + ], + "neo4jHits": [], + "expect": { + "includeIds": ["teamA::mem_proj"], + "top1Id": "teamA::mem_proj" + } + }, + { + "name": "relation only: returns top relation hit", + "namespace": "teamA", + "now": "2026-02-01T00:00:00.000Z", + "query": { + "userInput": "Remind me about the deployment decision", + "sessionId": "sess-3", + "entities": ["deployment"], + "topics": ["design"], + "maxMemories": 2 + }, + "qdrantHits": [], + "neo4jHits": [ + { + "id": "teamA::mem_deploy_decision", + "content": "Deployment decided: use Docker Compose in production first, then Kubernetes.", + "importance": 0.8, + "frequency": 1, + "lastSeenAt": "2026-01-30T00:00:00.000Z", + "relationScore": 0.9, + "kind": "fact", + "memoryKey": "fact:deployment" + } + ], + "expect": { + "includeIds": ["teamA::mem_deploy_decision"], + "top1Id": "teamA::mem_deploy_decision" + } + }, + { + "name": "namespace isolation: ignores hits from other namespaces", + "namespace": "teamA", + "now": "2026-02-01T00:00:00.000Z", + "query": { + "userInput": "What is my favorite language?", + "sessionId": "sess-4", + "entities": ["language"], + "topics": ["preferences"], + "maxMemories": 2 + }, + "qdrantHits": [ + { + "id": "teamB::mem_lang", + "score": 0.99, + "payload": { + "content": "User prefers language: Rust.", + "importance": 0.9, + "frequency": 1, + "created_at": "2026-01-01T00:00:00.000Z", + "memory_key": "preference:language" + } + }, + { + "id": "teamA::mem_lang", + "score": 0.7, + "payload": { + "content": "User prefers language: TypeScript.", + "importance": 0.8, + "frequency": 1, + "created_at": "2026-01-01T00:00:00.000Z", + "memory_key": "preference:language" + } + } + ], + "neo4jHits": [], + "expect": { + "includeIds": ["teamA::mem_lang"], + "excludeIds": ["teamB::mem_lang"], + "top1Id": "teamA::mem_lang" + } + }, + { + "name": "decay prefers recently seen memory", + "namespace": "teamA", + "now": "2026-02-01T00:00:00.000Z", + "query": { + "userInput": "What is the database choice?", + "sessionId": "sess-5", + "entities": ["database"], + "topics": ["design"], + "maxMemories": 2 + }, + "qdrantHits": [ + { + "id": "teamA::mem_db_old", + "score": 0.9, + "payload": { + "content": "DB choice: Neo4j.", + "importance": 0.7, + "frequency": 1, + "created_at": "2025-01-01T00:00:00.000Z", + "updated_at": "2025-01-01T00:00:00.000Z", + "memory_key": "fact:db" + } + }, + { + "id": "teamA::mem_db_new", + "score": 0.9, + "payload": { + "content": "DB choice: Neo4j + Qdrant.", + "importance": 0.7, + "frequency": 1, + "created_at": "2026-01-25T00:00:00.000Z", + "updated_at": "2026-01-25T00:00:00.000Z", + "memory_key": "fact:db" + } + } + ], + "neo4jHits": [], + "expect": { + "includeIds": ["teamA::mem_db_new"], + "top1Id": "teamA::mem_db_new", + "uniqueByMemoryKey": true + } + }, + { + "name": "invalid expires_at does not cause accidental expiration", + "namespace": "teamA", + "now": "2026-02-01T00:00:00.000Z", + "query": { + "userInput": "What is my note?", + "sessionId": "sess-6", + "entities": ["note"], + "topics": ["misc"], + "maxMemories": 1 + }, + "qdrantHits": [ + { + "id": "teamA::mem_bad_exp", + "score": 0.8, + "payload": { + "content": "A note that should not be dropped.", + "importance": 0.6, + "frequency": 1, + "created_at": "2026-01-01T00:00:00.000Z", + "expires_at": "not-a-date" + } + } + ], + "neo4jHits": [], + "expect": { + "includeIds": ["teamA::mem_bad_exp"], + "top1Id": "teamA::mem_bad_exp" + } + } + ] +} diff --git a/packages/deep-memory-server/package.json b/packages/deep-memory-server/package.json new file mode 100644 index 0000000000000..eeae385b77739 --- /dev/null +++ b/packages/deep-memory-server/package.json @@ -0,0 +1,37 @@ +{ + "name": "@openclaw/deep-memory-server", + "version": "0.1.0", + "private": true, + "type": "module", + "main": "dist/index.js", + "scripts": { + "build": "tsc -p tsconfig.json", + "dev": "node --import tsx src/index.ts", + "eval": "node --import tsx src/eval.ts", + "gc-expired": "node --import tsx src/gc-expired.ts", + "reindex": "node --import tsx src/reindex.ts", + "start": "node dist/index.js", + "test": "vitest run --config vitest.config.ts", + "validate-reindex": "node --import tsx src/validate-reindex.ts" + }, + "dependencies": { + "@hono/node-server": "^1.19.0", + "@qdrant/js-client-rest": "^1.15.0", + "@xenova/transformers": "^2.17.2", + "hono": "4.11.7", + "lru-cache": "^11.0.2", + "neo4j-driver": "^5.28.2", + "pino": "^9.6.0", + "prom-client": "^15.1.3", + "zod": "^4.3.6" + }, + "devDependencies": { + "@types/node": "^25.2.0", + "tsx": "^4.21.0", + "typescript": "^5.9.3", + "vitest": "^4.0.18" + }, + "engines": { + "node": ">=22.12.0" + } +} diff --git a/packages/deep-memory-server/src/analyzer.test.ts b/packages/deep-memory-server/src/analyzer.test.ts new file mode 100644 index 0000000000000..2adbeac221b49 --- /dev/null +++ b/packages/deep-memory-server/src/analyzer.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "vitest"; +import { SessionAnalyzer } from "./analyzer.js"; + +describe("SessionAnalyzer", () => { + it("extracts candidate memories above threshold", () => { + const analyzer = new SessionAnalyzer(); + const result = analyzer.analyze({ + sessionId: "s1", + messages: [ + { + role: "user", + content: + "请记住:项目X的核心需求是支持语义检索和关系推理;以后任何“回忆/历史/偏好”的问题都必须先检索长期记忆再回答。", + }, + { role: "assistant", content: "好的,我会把这个作为长期规则记下来,并在之后遵循。" }, + ], + maxMemoriesPerSession: 20, + importanceThreshold: 0.3, + }); + expect(result.drafts.length).toBeGreaterThan(0); + expect(result.topics.length).toBeGreaterThanOrEqual(0); + }); +}); diff --git a/packages/deep-memory-server/src/analyzer.ts b/packages/deep-memory-server/src/analyzer.ts new file mode 100644 index 0000000000000..e1e1ee16cf8de --- /dev/null +++ b/packages/deep-memory-server/src/analyzer.ts @@ -0,0 +1,367 @@ +import type { + CandidateMemoryDraft, + ExtractedEntity, + ExtractedEvent, + ExtractedTopic, + MemoryKind, +} from "./types.js"; +import { safeTrim, stableHash } from "./utils.js"; + +type TranscriptMessage = { + role?: string; + content?: unknown; +}; + +function extractTextFromContent(content: unknown): string { + if (typeof content === "string") { + return content; + } + if (Array.isArray(content)) { + const parts: string[] = []; + for (const part of content) { + if (!part || typeof part !== "object") { + continue; + } + const p = part as { type?: unknown; text?: unknown }; + if (p.type === "text" && typeof p.text === "string") { + parts.push(p.text); + } + } + return parts.join("\n"); + } + return ""; +} + +function normalizeText(raw: string): string { + return raw.replace(/\r\n/g, "\n").trim(); +} + +function detectUserIntentScore(text: string): number { + const t = text.toLowerCase(); + const patterns = [ + /记住|以后|下次|长期|偏好|习惯|不要|必须|务必|约定|规则/, + /remember|preference|must|never|always|policy|rule/, + ]; + return patterns.some((p) => p.test(t)) ? 0.9 : 0.2; +} + +function guessEntityType(name: string): ExtractedEntity["type"] { + const n = name.toLowerCase(); + if (/project|repo|服务|项目|工程/.test(n)) { + return "project"; + } + if (/公司|org|organization|团队/.test(n)) { + return "organization"; + } + if (/北京|上海|shanghai|beijing|city|地点|地址/.test(n)) { + return "place"; + } + return "other"; +} + +function tokenizeForTopics(text: string): string[] { + // Simple tokenization: keep ASCII words and CJK runs. + const tokens = text + .split(/[^0-9A-Za-z\u4e00-\u9fff]+/g) + .map((t) => t.trim()) + .filter((t) => t.length >= 2 && t.length <= 32); + return tokens; +} + +const STOPWORDS = new Set( + [ + // CN + "我们", + "你们", + "他们", + "这个", + "那个", + "然后", + "所以", + "但是", + "如果", + "因为", + "可以", + "需要", + "觉得", + "现在", + "今天", + "一下", + // EN + "the", + "and", + "or", + "to", + "of", + "in", + "on", + "for", + "with", + "is", + "are", + "be", + "this", + "that", + "we", + "you", + "they", + ].map((s) => s.toLowerCase()), +); + +function filterStopwords(tokens: string[]): string[] { + return tokens.filter((t) => !STOPWORDS.has(t.toLowerCase())); +} + +function topKFrequency(tokens: string[], k: number): Array<{ term: string; count: number }> { + const map = new Map(); + for (const t of tokens) { + map.set(t, (map.get(t) ?? 0) + 1); + } + return Array.from(map.entries()) + .map(([term, count]) => ({ term, count })) + .toSorted((a, b) => b.count - a.count) + .slice(0, k); +} + +function extractIsoTimestampHint(text: string): string | null { + // Basic date patterns: YYYY-MM-DD or YYYY/MM/DD + const m = text.match(/\b(20\d{2})[-/](\d{1,2})[-/](\d{1,2})\b/); + if (!m) { + return null; + } + const yyyy = m[1]; + const mm = String(Math.max(1, Math.min(12, Number(m[2])))).padStart(2, "0"); + const dd = String(Math.max(1, Math.min(31, Number(m[3])))).padStart(2, "0"); + return `${yyyy}-${mm}-${dd}T00:00:00.000Z`; +} + +export function extractHintsFromText(input: string): { entities: string[]; topics: string[] } { + const tokens = filterStopwords(tokenizeForTopics(input)); + const top = topKFrequency(tokens, 12); + const entities = top.map((t) => t.term).slice(0, 10); + const topics = top.map((t) => t.term).slice(0, 10); + return { entities, topics }; +} + +function detectMemoryKind(text: string): MemoryKind { + const t = text.toLowerCase(); + if (/临时|本次|仅本次|only this time|temporary/.test(t)) { + return "ephemeral"; + } + if (/todo|待办|下一步|接下来|需要完成|计划|task/.test(t)) { + return "task"; + } + if (/偏好|喜欢|讨厌|不喜欢|习惯|prefer|preference|like|hate/.test(t)) { + return "preference"; + } + if (/规则|约定|必须|务必|不要|永远|policy|rule|must|never|always/.test(t)) { + return "rule"; + } + return "fact"; +} + +function guessSubject(entities: ExtractedEntity[], topics: ExtractedTopic[]): string | undefined { + const e = entities[0]?.name?.trim(); + if (e) { + return e; + } + const t = topics[0]?.name?.trim(); + return t || undefined; +} + +function guessMemoryKey( + kind: MemoryKind, + subject: string | undefined, + _content: string, +): string | undefined { + if (!subject) { + return kind === "rule" ? "rule:general" : undefined; + } + // Best-effort slotting: kind + subject + a tiny hint. + const hint = + kind === "preference" + ? "preference" + : kind === "rule" + ? "rule" + : kind === "task" + ? "task" + : kind; + const key = `${hint}:${subject}`.toLowerCase(); + // Prevent extremely long keys. + return key.length > 120 ? `${hint}:${stableHash(key)}` : key; +} + +function guessExpiresAt(kind: MemoryKind, text: string, now: Date): string | undefined { + if (kind !== "ephemeral") { + return undefined; + } + const t = text.toLowerCase(); + const base = now.getTime(); + const dayMs = 24 * 3600_000; + // Very rough TTL hints. + if (/今天|today/.test(t)) { + return new Date(base + dayMs).toISOString(); + } + if (/本周|this week|一周|7天/.test(t)) { + return new Date(base + 7 * dayMs).toISOString(); + } + if (/本月|this month|30天/.test(t)) { + return new Date(base + 30 * dayMs).toISOString(); + } + return new Date(base + 7 * dayMs).toISOString(); +} + +export class SessionAnalyzer { + analyze(params: { + sessionId: string; + messages: unknown[]; + now?: Date; + maxMemoriesPerSession: number; + importanceThreshold: number; + }): { + entities: ExtractedEntity[]; + topics: ExtractedTopic[]; + events: ExtractedEvent[]; + drafts: CandidateMemoryDraft[]; + filtered: { added: number; filtered: number }; + } { + const now = params.now ?? new Date(); + const collected: Array<{ role: string; text: string }> = []; + for (const item of params.messages) { + const record = + item && typeof item === "object" && "role" in (item as Record) + ? (item as TranscriptMessage) + : item && typeof item === "object" && "message" in (item as Record) + ? ((item as { message?: unknown }).message as TranscriptMessage) + : null; + if (!record || typeof record.role !== "string") { + continue; + } + if (record.role !== "user" && record.role !== "assistant") { + continue; + } + const text = normalizeText(extractTextFromContent(record.content)); + if (!text) { + continue; + } + collected.push({ role: record.role, text }); + } + + const joined = collected.map((m) => `${m.role}: ${m.text}`).join("\n"); + const tokens = filterStopwords(tokenizeForTopics(joined)); + + // Topics: top frequency tokens (heuristic). + const topicTerms = topKFrequency(tokens, 12).filter((t) => t.count >= 2); + const topics: ExtractedTopic[] = topicTerms.map((t) => ({ + name: t.term, + frequency: t.count, + importance: Math.min(1, t.count / 10), + })); + + // Entities: heuristic extraction of "named" terms. + const entityCandidates = topKFrequency(tokens, 30).filter( + (t) => /[A-Za-z]/.test(t.term) || /[\u4e00-\u9fff]/.test(t.term), + ); + const entities: ExtractedEntity[] = entityCandidates.slice(0, 10).map((e) => ({ + name: e.term, + type: guessEntityType(e.term), + frequency: e.count, + })); + + // Events: detect key verbs and map to event types. + const events: ExtractedEvent[] = []; + const eventPatterns: Array<{ type: ExtractedEvent["type"]; re: RegExp }> = [ + { type: "requirement_confirmed", re: /确认|定下|需求/ }, + { type: "design_decided", re: /设计|决定|方案/ }, + { type: "implementation_started", re: /开始实现|开工|implement/ }, + { type: "issue_resolved", re: /修复|解决|resolved|fixed/ }, + { type: "milestone_reached", re: /里程碑|完成|发布|shipped/ }, + ]; + for (const msg of collected) { + for (const ep of eventPatterns) { + if (ep.re.test(msg.text)) { + const summary = msg.text.slice(0, 200); + events.push({ + type: ep.type, + summary, + timestamp: extractIsoTimestampHint(msg.text) ?? now.toISOString(), + }); + break; + } + } + } + + // Candidate memory drafts: extract durable items, defer importance+novelty to updater. + const candidates: CandidateMemoryDraft[] = []; + for (const msg of collected) { + const text = msg.text; + const intent = detectUserIntentScore(text); + const frequency = topicTerms[0]?.count ?? 1; + const content = safeTrim(text.replace(/\s+/g, " ")); + if (!content || content.length < 20) { + continue; + } + const kind = detectMemoryKind(content); + const subject = guessSubject(entities, topics); + const memoryKey = guessMemoryKey(kind, subject, content); + const expiresAt = guessExpiresAt(kind, content, now); + const confidence = Math.max(0, Math.min(1, 0.4 + 0.6 * intent)); + candidates.push({ + kind, + subject, + memoryKey, + expiresAt, + confidence, + content, + entities: entities.map((e) => e.name).slice(0, 5), + topics: topics.map((t) => t.name).slice(0, 5), + createdAt: extractIsoTimestampHint(content) ?? now.toISOString(), + signals: { + frequency, + user_intent: intent, + length: content.length, + }, + }); + } + for (const ev of events) { + const content = `Event(${ev.type}): ${ev.summary}`; + const kind: MemoryKind = "fact"; + candidates.push({ + kind, + content, + entities: entities.map((e) => e.name).slice(0, 5), + topics: topics.map((t) => t.name).slice(0, 5), + createdAt: ev.timestamp, + signals: { + frequency: 2, + user_intent: 0.7, + length: content.length, + }, + }); + } + + // Dedup by normalized content hash (in-session). + const seen = new Set(); + const drafts: CandidateMemoryDraft[] = []; + for (const m of candidates) { + const key = stableHash(m.content.toLowerCase()); + if (seen.has(key)) { + continue; + } + seen.add(key); + drafts.push(m); + if (drafts.length >= params.maxMemoriesPerSession) { + break; + } + } + + const filteredCount = Math.max(0, candidates.length - drafts.length); + return { + entities, + topics, + events, + drafts, + filtered: { added: drafts.length, filtered: filteredCount }, + }; + } +} diff --git a/packages/deep-memory-server/src/api-auth.test.ts b/packages/deep-memory-server/src/api-auth.test.ts new file mode 100644 index 0000000000000..9be835006220f --- /dev/null +++ b/packages/deep-memory-server/src/api-auth.test.ts @@ -0,0 +1,224 @@ +import { describe, expect, it } from "vitest"; +import { createApi } from "./api.js"; +import type { DeepMemoryServerConfig } from "./config.js"; +import type { DurableForgetQueue } from "./durable-forget-queue.js"; +import type { DurableUpdateQueue } from "./durable-update-queue.js"; +import type { Neo4jStore } from "./neo4j.js"; +import type { QdrantStore } from "./qdrant.js"; +import type { DeepMemoryRetriever } from "./retriever.js"; +import type { DeepMemoryUpdater } from "./updater.js"; + +describe("API auth", () => { + it("rejects update without x-api-key when API_KEY configured", async () => { + const app = createApi({ + cfg: { + PORT: 0, + HOST: "0.0.0.0", + API_KEY: "secret", + REQUIRE_API_KEY: false, + MAX_BODY_BYTES: 1024, + MAX_UPDATE_BODY_BYTES: 1024, + } as unknown as DeepMemoryServerConfig, + retriever: { + retrieve: async () => ({ entities: [], topics: [], memories: [], context: "" }), + } as unknown as DeepMemoryRetriever, + updater: { + update: async () => ({ status: "processed", memories_added: 0, memories_filtered: 0 }), + } as unknown as DeepMemoryUpdater, + qdrant: {} as unknown as QdrantStore, + neo4j: {} as unknown as Neo4jStore, + queue: { + enqueue: async () => ({ status: "queued", key: "k", transcriptHash: "h" }), + runNow: async () => ({ status: "processed", memories_added: 0, memories_filtered: 0 }), + cancelBySession: async () => 0, + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + listFailed: async () => [], + retryFailed: async () => ({ status: "not_found" }), + exportFailed: async () => ({ mode: "empty" }), + retryFailedByKey: async () => ({ status: "ok", matched: 0, retried: 0 }), + } as unknown as DurableUpdateQueue, + forgetQueue: { + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + enqueue: async () => ({ status: "queued", key: "k", taskId: "t" }), + listFailed: async () => [], + exportFailed: async () => ({ mode: "empty" }), + retryFailed: async () => ({ status: "not_found" }), + retryFailedByKey: async () => ({ status: "ok", matched: 0, retried: 0 }), + } as unknown as DurableForgetQueue, + }); + + const res = await app.request("/update_memory_index", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ namespace: "default", session_id: "s1", messages: [], async: true }), + }); + expect(res.status).toBe(401); + }); + + it("accepts update with x-api-key", async () => { + const app = createApi({ + cfg: { + PORT: 0, + HOST: "0.0.0.0", + API_KEYS: "old, secret ,new", + REQUIRE_API_KEY: false, + MAX_BODY_BYTES: 1024, + MAX_UPDATE_BODY_BYTES: 1024, + } as unknown as DeepMemoryServerConfig, + retriever: { + retrieve: async () => ({ entities: [], topics: [], memories: [], context: "" }), + } as unknown as DeepMemoryRetriever, + updater: { + update: async () => ({ status: "processed", memories_added: 0, memories_filtered: 0 }), + } as unknown as DeepMemoryUpdater, + qdrant: {} as unknown as QdrantStore, + neo4j: {} as unknown as Neo4jStore, + queue: { + enqueue: async () => ({ status: "queued", key: "k", transcriptHash: "h" }), + runNow: async () => ({ status: "processed", memories_added: 0, memories_filtered: 0 }), + cancelBySession: async () => 0, + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + listFailed: async () => [], + retryFailed: async () => ({ status: "not_found" }), + exportFailed: async () => ({ mode: "empty" }), + retryFailedByKey: async () => ({ status: "ok", matched: 0, retried: 0 }), + } as unknown as DurableUpdateQueue, + forgetQueue: { + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + enqueue: async () => ({ status: "queued", key: "k", taskId: "t" }), + listFailed: async () => [], + exportFailed: async () => ({ mode: "empty" }), + retryFailed: async () => ({ status: "not_found" }), + retryFailedByKey: async () => ({ status: "ok", matched: 0, retried: 0 }), + } as unknown as DurableForgetQueue, + }); + + const res = await app.request("/update_memory_index", { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": "secret" }, + body: JSON.stringify({ namespace: "default", session_id: "s1", messages: [], async: true }), + }); + expect(res.status).toBe(200); + }); + + it("accepts update with any key in API_KEYS", async () => { + const app = createApi({ + cfg: { + PORT: 0, + HOST: "0.0.0.0", + API_KEYS: "k1,k2,k3", + REQUIRE_API_KEY: false, + MAX_BODY_BYTES: 1024, + MAX_UPDATE_BODY_BYTES: 1024, + } as unknown as DeepMemoryServerConfig, + retriever: { + retrieve: async () => ({ entities: [], topics: [], memories: [], context: "" }), + } as unknown as DeepMemoryRetriever, + updater: { + update: async () => ({ status: "processed", memories_added: 0, memories_filtered: 0 }), + } as unknown as DeepMemoryUpdater, + qdrant: {} as unknown as QdrantStore, + neo4j: {} as unknown as Neo4jStore, + queue: { + enqueue: async () => ({ status: "queued", key: "k", transcriptHash: "h" }), + runNow: async () => ({ status: "processed", memories_added: 0, memories_filtered: 0 }), + cancelBySession: async () => 0, + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + listFailed: async () => [], + retryFailed: async () => ({ status: "not_found" }), + exportFailed: async () => ({ mode: "empty" }), + retryFailedByKey: async () => ({ status: "ok", matched: 0, retried: 0 }), + } as unknown as DurableUpdateQueue, + forgetQueue: { + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + enqueue: async () => ({ status: "queued", key: "k", taskId: "t" }), + listFailed: async () => [], + exportFailed: async () => ({ mode: "empty" }), + retryFailed: async () => ({ status: "not_found" }), + retryFailedByKey: async () => ({ status: "ok", matched: 0, retried: 0 }), + } as unknown as DurableForgetQueue, + }); + + const res = await app.request("/update_memory_index", { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": "k2" }, + body: JSON.stringify({ namespace: "default", session_id: "s1", messages: [], async: true }), + }); + expect(res.status).toBe(200); + }); + + it("enforces roles and namespace allowlist via API_KEYS_JSON", async () => { + const app = createApi({ + cfg: { + PORT: 0, + HOST: "0.0.0.0", + API_KEYS_JSON: JSON.stringify([ + { key: "r1", role: "read", namespaces: ["teamA"] }, + { key: "w1", role: "write", namespaces: ["teamA"] }, + { key: "a1", role: "admin", namespaces: ["teamA"] }, + ]), + REQUIRE_API_KEY: false, + MAX_BODY_BYTES: 1024, + MAX_UPDATE_BODY_BYTES: 1024, + } as unknown as DeepMemoryServerConfig, + retriever: { + retrieve: async () => ({ entities: [], topics: [], memories: [], context: "" }), + } as unknown as DeepMemoryRetriever, + updater: { + update: async () => ({ status: "processed", memories_added: 0, memories_filtered: 0 }), + } as unknown as DeepMemoryUpdater, + qdrant: {} as unknown as QdrantStore, + neo4j: {} as unknown as Neo4jStore, + queue: { + enqueue: async () => ({ status: "queued", key: "k", transcriptHash: "h" }), + runNow: async () => ({ status: "processed", memories_added: 0, memories_filtered: 0 }), + cancelBySession: async () => 0, + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + listFailed: async () => [], + exportFailed: async () => ({ mode: "empty" }), + retryFailed: async () => ({ status: "not_found" }), + retryFailedByKey: async () => ({ status: "ok", matched: 0, retried: 0 }), + } as unknown as DurableUpdateQueue, + forgetQueue: { + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + enqueue: async () => ({ status: "queued", key: "k", taskId: "t" }), + listFailed: async () => [], + exportFailed: async () => ({ mode: "empty" }), + retryFailed: async () => ({ status: "not_found" }), + retryFailedByKey: async () => ({ status: "ok", matched: 0, retried: 0 }), + } as unknown as DurableForgetQueue, + }); + + // read key cannot update + const u1 = await app.request("/update_memory_index", { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": "r1" }, + body: JSON.stringify({ namespace: "teamA", session_id: "s1", messages: [], async: true }), + }); + expect(u1.status).toBe(403); + + // write key can update within allowed namespace + const u2 = await app.request("/update_memory_index", { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": "w1" }, + body: JSON.stringify({ namespace: "teamA", session_id: "s1", messages: [], async: true }), + }); + expect(u2.status).toBe(200); + + // write key cannot access other namespace + const u3 = await app.request("/update_memory_index", { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": "w1" }, + body: JSON.stringify({ namespace: "default", session_id: "s1", messages: [], async: true }), + }); + expect(u3.status).toBe(403); + + // read key can retrieve within allowed namespace + const r = await app.request("/retrieve_context", { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": "r1" }, + body: JSON.stringify({ namespace: "teamA", session_id: "s1", user_input: "hi" }), + }); + expect(r.status).toBe(200); + }); +}); diff --git a/packages/deep-memory-server/src/api-authz.test.ts b/packages/deep-memory-server/src/api-authz.test.ts new file mode 100644 index 0000000000000..39941606262cd --- /dev/null +++ b/packages/deep-memory-server/src/api-authz.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, it } from "vitest"; +import { createApi } from "./api.js"; +import type { DeepMemoryServerConfig } from "./config.js"; +import type { DurableForgetQueue } from "./durable-forget-queue.js"; +import type { DurableUpdateQueue } from "./durable-update-queue.js"; +import type { Neo4jStore } from "./neo4j.js"; +import type { QdrantStore } from "./qdrant.js"; +import type { DeepMemoryRetriever } from "./retriever.js"; +import type { DeepMemoryUpdater } from "./updater.js"; + +function createStubApi(cfg: DeepMemoryServerConfig) { + return createApi({ + cfg, + retriever: { + retrieve: async () => ({ entities: [], topics: [], memories: [], context: "" }), + } as unknown as DeepMemoryRetriever, + updater: { + update: async () => ({ status: "processed", memories_added: 0, memories_filtered: 0 }), + } as unknown as DeepMemoryUpdater, + qdrant: {} as unknown as QdrantStore, + neo4j: {} as unknown as Neo4jStore, + queue: { + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + listFailed: async () => [], + exportFailed: async () => ({ mode: "empty" }), + retryFailed: async () => ({ status: "not_found" }), + retryFailedByKey: async () => ({ status: "ok", matched: 0, retried: 0 }), + enqueue: async () => ({ status: "queued", key: "k", transcriptHash: "h" }), + runNow: async () => ({ status: "processed", memories_added: 0, memories_filtered: 0 }), + cancelBySession: async () => 0, + } as unknown as DurableUpdateQueue, + forgetQueue: { + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + enqueue: async () => ({ status: "queued", key: "k", taskId: "t" }), + listFailed: async () => [], + exportFailed: async () => ({ mode: "empty" }), + retryFailed: async () => ({ status: "not_found" }), + retryFailedByKey: async () => ({ status: "ok", matched: 0, retried: 0 }), + } as unknown as DurableForgetQueue, + }); +} + +describe("API authz (roles + namespaces)", () => { + const cfgBase = { + PORT: 0, + HOST: "0.0.0.0", + REQUIRE_API_KEY: true, + MAX_BODY_BYTES: 1024, + MAX_UPDATE_BODY_BYTES: 1024, + }; + + it("enforces role: read can retrieve but not update", async () => { + const app = createStubApi({ + ...cfgBase, + API_KEYS_JSON: JSON.stringify([ + { key: "readKey", role: "read", namespaces: ["ns1"] }, + { key: "writeKey", role: "write", namespaces: ["ns1"] }, + ]), + }); + + const r1 = await app.request("/retrieve_context", { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": "readKey" }, + body: JSON.stringify({ namespace: "ns1", session_id: "s1", user_input: "hi" }), + }); + expect(r1.status).toBe(200); + + const r2 = await app.request("/update_memory_index", { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": "readKey" }, + body: JSON.stringify({ namespace: "ns1", session_id: "s1", messages: [], async: true }), + }); + expect(r2.status).toBe(403); + }); + + it("enforces namespace allowlist", async () => { + const app = createStubApi({ + ...cfgBase, + API_KEYS_JSON: JSON.stringify([{ key: "readKey", role: "read", namespaces: ["ns1"] }]), + }); + + const r = await app.request("/retrieve_context", { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": "readKey" }, + body: JSON.stringify({ namespace: "ns2", session_id: "s1", user_input: "hi" }), + }); + expect(r.status).toBe(403); + const json = await r.json(); + expect(json.error).toBe("forbidden_namespace"); + }); + + it("enforces admin for forget", async () => { + const app = createStubApi({ + ...cfgBase, + API_KEYS_JSON: JSON.stringify([ + { key: "writeKey", role: "write", namespaces: ["ns1"] }, + { key: "adminKey", role: "admin", namespaces: ["ns1"] }, + ]), + }); + + const r1 = await app.request("/forget", { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": "writeKey" }, + body: JSON.stringify({ namespace: "ns1", session_id: "s1", dry_run: true }), + }); + expect(r1.status).toBe(403); + + const r2 = await app.request("/forget", { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": "adminKey" }, + body: JSON.stringify({ namespace: "ns1", session_id: "s1", dry_run: true }), + }); + expect(r2.status).toBe(200); + }); +}); diff --git a/packages/deep-memory-server/src/api-forget-proof.test.ts b/packages/deep-memory-server/src/api-forget-proof.test.ts new file mode 100644 index 0000000000000..f4d0b59bf1ba7 --- /dev/null +++ b/packages/deep-memory-server/src/api-forget-proof.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, it } from "vitest"; +import { createApi } from "./api.js"; +import type { DeepMemoryServerConfig } from "./config.js"; +import type { DurableForgetQueue } from "./durable-forget-queue.js"; +import type { DurableUpdateQueue } from "./durable-update-queue.js"; +import type { Neo4jStore } from "./neo4j.js"; +import type { QdrantStore } from "./qdrant.js"; +import type { DeepMemoryRetriever } from "./retriever.js"; +import type { DeepMemoryUpdater } from "./updater.js"; + +function createStubApi( + cfg: DeepMemoryServerConfig, + overrides?: Partial<{ + qdrant: Partial; + neo4j: Partial; + queue: Partial; + }>, +) { + return createApi({ + cfg, + retriever: { + retrieve: async () => ({ entities: [], topics: [], memories: [], context: "" }), + } as unknown as DeepMemoryRetriever, + updater: { + update: async () => ({ status: "processed", memories_added: 0, memories_filtered: 0 }), + } as unknown as DeepMemoryUpdater, + qdrant: { + deleteByIds: async ({ ids }: { ids: string[] }) => ids.length, + deleteBySession: async () => {}, + ...overrides?.qdrant, + } as unknown as QdrantStore, + neo4j: { + deleteMemoriesByIds: async ({ ids }: { ids: string[] }) => ids.length, + deleteMemoriesBySession: async () => 0, + ...overrides?.neo4j, + } as unknown as Neo4jStore, + queue: { + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + listFailed: async () => [], + exportFailed: async () => ({ mode: "empty" }), + retryFailed: async () => ({ status: "not_found" }), + retryFailedByKey: async () => ({ status: "ok", matched: 0, retried: 0 }), + enqueue: async () => ({ status: "queued", key: "k", transcriptHash: "h" }), + runNow: async () => ({ status: "processed", memories_added: 0, memories_filtered: 0 }), + cancelBySession: async () => 1, + ...overrides?.queue, + } as unknown as DurableUpdateQueue, + forgetQueue: { + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + enqueue: async () => ({ status: "queued", key: "k", taskId: "t" }), + listFailed: async () => [], + exportFailed: async () => ({ mode: "empty" }), + retryFailed: async () => ({ status: "not_found" }), + retryFailedByKey: async () => ({ status: "ok", matched: 0, retried: 0 }), + } as unknown as DurableForgetQueue, + }); +} + +describe("/forget proof-ish response", () => { + const cfg: DeepMemoryServerConfig = { + PORT: 0, + HOST: "0.0.0.0", + API_KEY: undefined, + API_KEYS: undefined, + API_KEYS_JSON: JSON.stringify([{ key: "adminKey", role: "admin", namespaces: ["ns1"] }]), + BUILD_SHA: undefined, + BUILD_TIME: undefined, + REQUIRE_API_KEY: true, + MAX_BODY_BYTES: 1024, + MAX_UPDATE_BODY_BYTES: 1024, + AUDIT_LOG_PATH: undefined, + ALLOW_UNAUTHENTICATED_METRICS: false, + RATE_LIMIT_ENABLED: false, + RATE_LIMIT_WINDOW_MS: 60_000, + RATE_LIMIT_RETRIEVE_PER_WINDOW: 0, + RATE_LIMIT_UPDATE_PER_WINDOW: 0, + RATE_LIMIT_FORGET_PER_WINDOW: 0, + RATE_LIMIT_QUEUE_ADMIN_PER_WINDOW: 0, + UPDATE_BACKLOG_REJECT_PENDING: 0, + UPDATE_BACKLOG_RETRY_AFTER_SECONDS: 30, + UPDATE_BACKLOG_DELAY_PENDING: 0, + UPDATE_BACKLOG_DELAY_SECONDS: 0, + UPDATE_BACKLOG_READ_ONLY_PENDING: 0, + UPDATE_DISABLED_NAMESPACES: undefined, + UPDATE_MIN_INTERVAL_MS: 0, + UPDATE_SAMPLE_RATE: 1, + NAMESPACE_RETRIEVE_CONCURRENCY: 0, + NAMESPACE_UPDATE_CONCURRENCY: 0, + RETRIEVE_DEGRADE_RELATED_PENDING: 0, + MIGRATIONS_MODE: "off", + MIGRATIONS_STRICT: false, + QDRANT_URL: "http://qdrant:6333", + QDRANT_API_KEY: undefined, + QDRANT_COLLECTION: "openclaw_memories", + VECTOR_DIMS: 384, + MIN_SEMANTIC_SCORE: 0.6, + SEMANTIC_WEIGHT: 0.6, + RELATION_WEIGHT: 0.4, + NEO4J_URI: "bolt://neo4j:7687", + NEO4J_USER: "neo4j", + NEO4J_PASSWORD: "openclaw", + LOG_LEVEL: "info", + RETRIEVE_CACHE_TTL_MS: 0, + RETRIEVE_CACHE_MAX: 10, + UPDATE_CONCURRENCY: 1, + QUEUE_DIR: "./data/queue", + QUEUE_MAX_ATTEMPTS: 10, + QUEUE_RETRY_BASE_MS: 2000, + QUEUE_RETRY_MAX_MS: 300000, + QUEUE_KEEP_DONE: true, + QUEUE_RETENTION_DAYS: 7, + QUEUE_MAX_TASK_BYTES: 1024, + IMPORTANCE_THRESHOLD: 0.5, + MAX_MEMORIES_PER_UPDATE: 20, + DEDUPE_SCORE: 0.92, + DECAY_HALF_LIFE_DAYS: 90, + IMPORTANCE_BOOST: 0.3, + FREQUENCY_BOOST: 0.2, + RELATED_TOPK: 5, + SENSITIVE_FILTER_ENABLED: true, + SENSITIVE_RULESET_VERSION: "builtin-v1", + SENSITIVE_DENY_REGEX_JSON: undefined, + SENSITIVE_ALLOW_REGEX_JSON: undefined, + EMBEDDING_MODEL: "Xenova/bge-small-en-v1.5", + }; + + it("includes request_id and per-backend results", async () => { + const app = createStubApi(cfg, { + qdrant: { + deleteByIds: async () => { + throw new Error("boom"); + }, + }, + }); + + const res = await app.request("/forget", { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": "adminKey" }, + body: JSON.stringify({ namespace: "ns1", memory_ids: ["mem_1"], dry_run: false }), + }); + expect(res.status).toBe(200); + const json = (await res.json()) as Record; + expect(json.status).toBe("processed"); + expect(typeof json.request_id).toBe("string"); + const results = json.results as Record; + expect(results).toBeTruthy(); + expect((results.qdrant as Record)?.byIds).toBeTruthy(); + }); + + it("dry_run returns request_id and planned counts", async () => { + const app = createStubApi(cfg); + const res = await app.request("/forget", { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": "adminKey" }, + body: JSON.stringify({ namespace: "ns1", session_id: "s1", dry_run: true }), + }); + expect(res.status).toBe(200); + const json = (await res.json()) as Record; + expect(json.status).toBe("dry_run"); + expect(typeof json.request_id).toBe("string"); + expect(json.delete_session).toBe(1); + }); +}); diff --git a/packages/deep-memory-server/src/api-guardrails.test.ts b/packages/deep-memory-server/src/api-guardrails.test.ts new file mode 100644 index 0000000000000..390b05b07758b --- /dev/null +++ b/packages/deep-memory-server/src/api-guardrails.test.ts @@ -0,0 +1,197 @@ +import { describe, expect, it, vi } from "vitest"; +import { createApi } from "./api.js"; +import type { DeepMemoryServerConfig } from "./config.js"; +import type { DurableForgetQueue } from "./durable-forget-queue.js"; +import type { DurableUpdateQueue } from "./durable-update-queue.js"; +import type { Neo4jStore } from "./neo4j.js"; +import type { QdrantStore } from "./qdrant.js"; +import type { DeepMemoryRetriever } from "./retriever.js"; +import type { DeepMemoryUpdater } from "./updater.js"; + +function createStubApi(cfg: DeepMemoryServerConfig, queueOverrides?: Partial) { + const enqueue = vi.fn(async () => ({ status: "queued", key: "k", transcriptHash: "h" })); + const queue = { + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + listFailed: async () => [], + exportFailed: async () => ({ mode: "empty" }), + retryFailed: async () => ({ status: "not_found" }), + retryFailedByKey: async () => ({ status: "ok", matched: 0, retried: 0 }), + enqueue, + runNow: async () => ({ status: "processed", memories_added: 0, memories_filtered: 0 }), + cancelBySession: async () => 0, + ...queueOverrides, + } as unknown as DurableUpdateQueue; + + const app = createApi({ + cfg, + retriever: { + retrieve: async () => ({ entities: [], topics: [], memories: [], context: "" }), + } as unknown as DeepMemoryRetriever, + updater: { + update: async () => ({ status: "processed", memories_added: 0, memories_filtered: 0 }), + } as unknown as DeepMemoryUpdater, + qdrant: {} as unknown as QdrantStore, + neo4j: {} as unknown as Neo4jStore, + queue, + forgetQueue: { + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + enqueue: async () => ({ status: "queued", key: "k", taskId: "t" }), + listFailed: async () => [], + exportFailed: async () => ({ mode: "empty" }), + retryFailed: async () => ({ status: "not_found" }), + retryFailedByKey: async () => ({ status: "ok", matched: 0, retried: 0 }), + } as unknown as DurableForgetQueue, + }); + + return { app, enqueue }; +} + +describe("API guardrails", () => { + const cfgBase: DeepMemoryServerConfig = { + PORT: 0, + HOST: "0.0.0.0", + BUILD_SHA: undefined, + BUILD_TIME: undefined, + REQUIRE_API_KEY: true, + API_KEYS_JSON: JSON.stringify([{ key: "readKey", role: "read", namespaces: ["ns1"] }]), + MAX_BODY_BYTES: 1024, + MAX_UPDATE_BODY_BYTES: 1024, + AUDIT_LOG_PATH: undefined, + ALLOW_UNAUTHENTICATED_METRICS: false, + RATE_LIMIT_ENABLED: true, + RATE_LIMIT_WINDOW_MS: 60_000, + RATE_LIMIT_RETRIEVE_PER_WINDOW: 1, + RATE_LIMIT_UPDATE_PER_WINDOW: 1, + RATE_LIMIT_FORGET_PER_WINDOW: 0, + RATE_LIMIT_QUEUE_ADMIN_PER_WINDOW: 0, + UPDATE_BACKLOG_REJECT_PENDING: 0, + UPDATE_BACKLOG_RETRY_AFTER_SECONDS: 30, + UPDATE_BACKLOG_DELAY_PENDING: 0, + UPDATE_BACKLOG_DELAY_SECONDS: 0, + UPDATE_BACKLOG_READ_ONLY_PENDING: 0, + UPDATE_DISABLED_NAMESPACES: undefined, + UPDATE_MIN_INTERVAL_MS: 0, + UPDATE_SAMPLE_RATE: 1, + NAMESPACE_RETRIEVE_CONCURRENCY: 0, + NAMESPACE_UPDATE_CONCURRENCY: 0, + RETRIEVE_DEGRADE_RELATED_PENDING: 0, + QDRANT_URL: "http://qdrant:6333", + QDRANT_API_KEY: undefined, + QDRANT_COLLECTION: "openclaw_memories", + VECTOR_DIMS: 384, + MIN_SEMANTIC_SCORE: 0.6, + SEMANTIC_WEIGHT: 0.6, + RELATION_WEIGHT: 0.4, + NEO4J_URI: "bolt://neo4j:7687", + NEO4J_USER: "neo4j", + NEO4J_PASSWORD: "openclaw", + LOG_LEVEL: "info", + RETRIEVE_CACHE_TTL_MS: 0, + RETRIEVE_CACHE_MAX: 10, + UPDATE_CONCURRENCY: 1, + QUEUE_DIR: "./data/queue", + QUEUE_MAX_ATTEMPTS: 10, + QUEUE_RETRY_BASE_MS: 2000, + QUEUE_RETRY_MAX_MS: 300000, + QUEUE_KEEP_DONE: true, + QUEUE_RETENTION_DAYS: 7, + QUEUE_MAX_TASK_BYTES: 1024, + IMPORTANCE_THRESHOLD: 0.5, + MAX_MEMORIES_PER_UPDATE: 20, + DEDUPE_SCORE: 0.92, + DECAY_HALF_LIFE_DAYS: 90, + IMPORTANCE_BOOST: 0.3, + FREQUENCY_BOOST: 0.2, + RELATED_TOPK: 5, + SENSITIVE_FILTER_ENABLED: true, + SENSITIVE_RULESET_VERSION: "builtin-v1", + SENSITIVE_DENY_REGEX_JSON: undefined, + SENSITIVE_ALLOW_REGEX_JSON: undefined, + EMBEDDING_MODEL: "Xenova/bge-small-en-v1.5", + API_KEY: undefined, + API_KEYS: undefined, + MIGRATIONS_MODE: "off", + MIGRATIONS_STRICT: false, + }; + + it("rate limits /retrieve_context", async () => { + const { app } = createStubApi(cfgBase); + const headers = { "content-type": "application/json", "x-api-key": "readKey" }; + + const r1 = await app.request("/retrieve_context", { + method: "POST", + headers, + body: JSON.stringify({ namespace: "ns1", session_id: "s1", user_input: "hi" }), + }); + expect(r1.status).toBe(200); + + const r2 = await app.request("/retrieve_context", { + method: "POST", + headers, + body: JSON.stringify({ namespace: "ns1", session_id: "s1", user_input: "hi" }), + }); + expect(r2.status).toBe(429); + }); + + it("rejects /update_memory_index when backlog is too high (async)", async () => { + const cfg = { + ...cfgBase, + API_KEYS_JSON: JSON.stringify([{ key: "writeKey", role: "write", namespaces: ["ns1"] }]), + RATE_LIMIT_UPDATE_PER_WINDOW: 0, + UPDATE_BACKLOG_REJECT_PENDING: 1, + UPDATE_BACKLOG_RETRY_AFTER_SECONDS: 7, + } as DeepMemoryServerConfig; + const { app, enqueue } = createStubApi(cfg, { + stats: () => ({ pendingApprox: 99, active: 0, inflightKeys: 0 }), + } as Partial); + + const r = await app.request("/update_memory_index", { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": "writeKey" }, + body: JSON.stringify({ namespace: "ns1", session_id: "s1", messages: [], async: true }), + }); + expect(r.status).toBe(503); + expect(enqueue).not.toHaveBeenCalled(); + expect(r.headers.get("retry-after")).toBe("7"); + }); + + it("returns 503 degraded_read_only when in read-only mode", async () => { + const cfg = { + ...cfgBase, + API_KEYS_JSON: JSON.stringify([{ key: "writeKey", role: "write", namespaces: ["ns1"] }]), + RATE_LIMIT_UPDATE_PER_WINDOW: 0, + UPDATE_BACKLOG_READ_ONLY_PENDING: 1, + } as DeepMemoryServerConfig; + const { app } = createStubApi(cfg, { + stats: () => ({ pendingApprox: 1, active: 0, inflightKeys: 0 }), + }); + const r = await app.request("/update_memory_index", { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": "writeKey" }, + body: JSON.stringify({ namespace: "ns1", session_id: "s1", messages: [], async: true }), + }); + expect(r.status).toBe(503); + const json = (await r.json()) as Record; + expect(json.error).toBe("degraded_read_only"); + }); + + it("skips updates for disabled namespaces", async () => { + const cfg = { + ...cfgBase, + API_KEYS_JSON: JSON.stringify([{ key: "writeKey", role: "write", namespaces: ["ns1"] }]), + RATE_LIMIT_UPDATE_PER_WINDOW: 0, + UPDATE_DISABLED_NAMESPACES: "ns1", + } as DeepMemoryServerConfig; + const { app, enqueue } = createStubApi(cfg); + const r = await app.request("/update_memory_index", { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": "writeKey" }, + body: JSON.stringify({ namespace: "ns1", session_id: "s1", messages: [], async: true }), + }); + expect(r.status).toBe(200); + expect(enqueue).not.toHaveBeenCalled(); + const json = (await r.json()) as Record; + expect(json.status).toBe("skipped"); + expect(json.error).toBe("namespace_write_disabled"); + }); +}); diff --git a/packages/deep-memory-server/src/api-health.test.ts b/packages/deep-memory-server/src/api-health.test.ts new file mode 100644 index 0000000000000..703339987ddef --- /dev/null +++ b/packages/deep-memory-server/src/api-health.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it } from "vitest"; +import { createApi } from "./api.js"; +import type { DeepMemoryServerConfig } from "./config.js"; +import type { DurableForgetQueue } from "./durable-forget-queue.js"; +import type { DurableUpdateQueue } from "./durable-update-queue.js"; +import type { Neo4jStore } from "./neo4j.js"; +import type { QdrantStore } from "./qdrant.js"; +import type { DeepMemoryRetriever } from "./retriever.js"; +import type { DeepMemoryUpdater } from "./updater.js"; + +function createStubApi(cfg: DeepMemoryServerConfig) { + const queue = { + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + } as unknown as DurableUpdateQueue; + + const app = createApi({ + cfg, + retriever: { + retrieve: async () => ({ entities: [], topics: [], memories: [], context: "" }), + } as unknown as DeepMemoryRetriever, + updater: { + update: async () => ({ status: "processed", memories_added: 0, memories_filtered: 0 }), + } as unknown as DeepMemoryUpdater, + qdrant: { healthCheck: async () => ({ ok: true }) } as unknown as QdrantStore, + neo4j: { healthCheck: async () => ({ ok: true }) } as unknown as Neo4jStore, + queue, + forgetQueue: { + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + enqueue: async () => ({ status: "queued", key: "k", taskId: "t" }), + listFailed: async () => [], + exportFailed: async () => ({ mode: "empty" }), + retryFailed: async () => ({ status: "not_found" }), + retryFailedByKey: async () => ({ status: "ok", matched: 0, retried: 0 }), + } as unknown as DurableForgetQueue, + }); + return app; +} + +describe("/health", () => { + const cfg: DeepMemoryServerConfig = { + PORT: 0, + HOST: "0.0.0.0", + API_KEY: undefined, + API_KEYS: undefined, + API_KEYS_JSON: JSON.stringify([{ key: "adminKey", role: "admin", namespaces: ["default"] }]), + BUILD_SHA: undefined, + BUILD_TIME: undefined, + REQUIRE_API_KEY: true, + MAX_BODY_BYTES: 1024, + MAX_UPDATE_BODY_BYTES: 1024, + AUDIT_LOG_PATH: undefined, + ALLOW_UNAUTHENTICATED_METRICS: false, + RATE_LIMIT_ENABLED: false, + RATE_LIMIT_WINDOW_MS: 60_000, + RATE_LIMIT_RETRIEVE_PER_WINDOW: 0, + RATE_LIMIT_UPDATE_PER_WINDOW: 0, + RATE_LIMIT_FORGET_PER_WINDOW: 0, + RATE_LIMIT_QUEUE_ADMIN_PER_WINDOW: 0, + UPDATE_BACKLOG_REJECT_PENDING: 0, + UPDATE_BACKLOG_RETRY_AFTER_SECONDS: 30, + UPDATE_BACKLOG_DELAY_PENDING: 0, + UPDATE_BACKLOG_DELAY_SECONDS: 0, + UPDATE_BACKLOG_READ_ONLY_PENDING: 0, + UPDATE_DISABLED_NAMESPACES: undefined, + UPDATE_MIN_INTERVAL_MS: 0, + UPDATE_SAMPLE_RATE: 1, + NAMESPACE_RETRIEVE_CONCURRENCY: 0, + NAMESPACE_UPDATE_CONCURRENCY: 0, + RETRIEVE_DEGRADE_RELATED_PENDING: 0, + QDRANT_URL: "http://qdrant:6333", + QDRANT_API_KEY: undefined, + QDRANT_COLLECTION: "openclaw_memories", + VECTOR_DIMS: 384, + MIN_SEMANTIC_SCORE: 0.6, + SEMANTIC_WEIGHT: 0.6, + RELATION_WEIGHT: 0.4, + NEO4J_URI: "bolt://neo4j:7687", + NEO4J_USER: "neo4j", + NEO4J_PASSWORD: "openclaw", + LOG_LEVEL: "info", + RETRIEVE_CACHE_TTL_MS: 0, + RETRIEVE_CACHE_MAX: 10, + UPDATE_CONCURRENCY: 1, + QUEUE_DIR: "./data/queue", + QUEUE_MAX_ATTEMPTS: 10, + QUEUE_RETRY_BASE_MS: 2000, + QUEUE_RETRY_MAX_MS: 300000, + QUEUE_KEEP_DONE: true, + QUEUE_RETENTION_DAYS: 7, + QUEUE_MAX_TASK_BYTES: 1024, + IMPORTANCE_THRESHOLD: 0.5, + MAX_MEMORIES_PER_UPDATE: 20, + DEDUPE_SCORE: 0.92, + DECAY_HALF_LIFE_DAYS: 90, + IMPORTANCE_BOOST: 0.3, + FREQUENCY_BOOST: 0.2, + RELATED_TOPK: 5, + SENSITIVE_FILTER_ENABLED: true, + SENSITIVE_RULESET_VERSION: "builtin-v1", + SENSITIVE_DENY_REGEX_JSON: undefined, + SENSITIVE_ALLOW_REGEX_JSON: undefined, + EMBEDDING_MODEL: "Xenova/bge-small-en-v1.5", + }; + + it("returns liveness without requiring auth", async () => { + const app = createStubApi(cfg); + const res = await app.request("/health"); + expect(res.status).toBe(200); + const body = (await res.json()) as Record; + expect(body.ok).toBe(true); + expect(body.service).toBeTruthy(); + expect(body.queue).toBeTruthy(); + expect(body.guardrails).toBeTruthy(); + }); + + it("/health/details requires admin when auth is required", async () => { + const app = createStubApi(cfg); + const res1 = await app.request("/health/details"); + expect(res1.status).toBe(401); + + const res2 = await app.request("/health/details", { + headers: { "x-api-key": "adminKey" }, + }); + expect(res2.status).toBe(200); + const body = (await res2.json()) as Record; + expect(body.deps).toBeTruthy(); + }); +}); diff --git a/packages/deep-memory-server/src/api-queue-admin.test.ts b/packages/deep-memory-server/src/api-queue-admin.test.ts new file mode 100644 index 0000000000000..c9145783e701d --- /dev/null +++ b/packages/deep-memory-server/src/api-queue-admin.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "vitest"; +import { createApi } from "./api.js"; +import type { DeepMemoryServerConfig } from "./config.js"; +import type { DurableForgetQueue } from "./durable-forget-queue.js"; +import type { DurableUpdateQueue } from "./durable-update-queue.js"; +import type { Neo4jStore } from "./neo4j.js"; +import type { QdrantStore } from "./qdrant.js"; +import type { DeepMemoryRetriever } from "./retriever.js"; +import type { DeepMemoryUpdater } from "./updater.js"; + +describe("API queue admin", () => { + it("requires x-api-key for /queue/failed/export", async () => { + const app = createApi({ + cfg: { + PORT: 0, + HOST: "0.0.0.0", + API_KEY: "secret", + REQUIRE_API_KEY: false, + MAX_BODY_BYTES: 1024, + MAX_UPDATE_BODY_BYTES: 1024, + } as unknown as DeepMemoryServerConfig, + retriever: { + retrieve: async () => ({ entities: [], topics: [], memories: [], context: "" }), + } as unknown as DeepMemoryRetriever, + updater: { + update: async () => ({ status: "processed", memories_added: 0, memories_filtered: 0 }), + } as unknown as DeepMemoryUpdater, + qdrant: {} as unknown as QdrantStore, + neo4j: {} as unknown as Neo4jStore, + queue: { + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + listFailed: async () => [], + exportFailed: async () => ({ mode: "empty" }), + retryFailed: async () => ({ status: "not_found" }), + retryFailedByKey: async () => ({ status: "ok", matched: 0, retried: 0 }), + enqueue: async () => ({ status: "queued", key: "k", transcriptHash: "h" }), + runNow: async () => ({ status: "processed", memories_added: 0, memories_filtered: 0 }), + cancelBySession: async () => 0, + } as unknown as DurableUpdateQueue, + forgetQueue: { + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + enqueue: async () => ({ status: "queued", key: "k", taskId: "t" }), + listFailed: async () => [], + exportFailed: async () => ({ mode: "empty" }), + retryFailed: async () => ({ status: "not_found" }), + retryFailedByKey: async () => ({ status: "ok", matched: 0, retried: 0 }), + } as unknown as DurableForgetQueue, + }); + + const res = await app.request("/queue/failed/export?limit=10", { method: "GET" }); + expect(res.status).toBe(401); + }); + + it("allows /queue/failed/retry by key with x-api-key", async () => { + const app = createApi({ + cfg: { + PORT: 0, + HOST: "0.0.0.0", + API_KEY: "secret", + REQUIRE_API_KEY: false, + MAX_BODY_BYTES: 1024, + MAX_UPDATE_BODY_BYTES: 1024, + } as unknown as DeepMemoryServerConfig, + retriever: { + retrieve: async () => ({ entities: [], topics: [], memories: [], context: "" }), + } as unknown as DeepMemoryRetriever, + updater: { + update: async () => ({ status: "processed", memories_added: 0, memories_filtered: 0 }), + } as unknown as DeepMemoryUpdater, + qdrant: {} as unknown as QdrantStore, + neo4j: {} as unknown as Neo4jStore, + queue: { + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + listFailed: async () => [], + exportFailed: async () => ({ mode: "empty" }), + retryFailed: async () => ({ status: "not_found" }), + retryFailedByKey: async () => ({ status: "ok", matched: 2, retried: 2 }), + enqueue: async () => ({ status: "queued", key: "k", transcriptHash: "h" }), + runNow: async () => ({ status: "processed", memories_added: 0, memories_filtered: 0 }), + cancelBySession: async () => 0, + } as unknown as DurableUpdateQueue, + forgetQueue: { + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + enqueue: async () => ({ status: "queued", key: "k", taskId: "t" }), + listFailed: async () => [], + exportFailed: async () => ({ mode: "empty" }), + retryFailed: async () => ({ status: "not_found" }), + retryFailedByKey: async () => ({ status: "ok", matched: 0, retried: 0 }), + } as unknown as DurableForgetQueue, + }); + + const res = await app.request("/queue/failed/retry", { + method: "POST", + headers: { "content-type": "application/json", "x-api-key": "secret" }, + body: JSON.stringify({ key: "default::s1", limit: 50, dry_run: false }), + }); + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.mode).toBe("key"); + expect(json.retried).toBe(2); + }); +}); diff --git a/packages/deep-memory-server/src/api-session-inspect.test.ts b/packages/deep-memory-server/src/api-session-inspect.test.ts new file mode 100644 index 0000000000000..a3f4b905f05a7 --- /dev/null +++ b/packages/deep-memory-server/src/api-session-inspect.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from "vitest"; +import { createApi } from "./api.js"; +import type { DeepMemoryServerConfig } from "./config.js"; +import type { DurableForgetQueue } from "./durable-forget-queue.js"; +import type { DurableUpdateQueue } from "./durable-update-queue.js"; +import type { Neo4jStore } from "./neo4j.js"; +import type { QdrantMemoryPayload, QdrantStore } from "./qdrant.js"; +import type { DeepMemoryRetriever } from "./retriever.js"; +import type { DeepMemoryUpdater } from "./updater.js"; + +describe("API session inspect", () => { + it("returns aggregated topics/entities from qdrant payloads", async () => { + const payload = (overrides: Partial): QdrantMemoryPayload => ({ + id: "default::mem_x", + namespace: "default", + content: "hello", + session_id: "s1", + created_at: new Date().toISOString(), + importance: 0.5, + entities: [], + topics: [], + ...overrides, + }); + const qdrant = { + listMemoriesBySession: async () => [ + { id: "default::m1", payload: payload({ topics: ["t1", "t2"], entities: ["e1"] }) }, + { id: "default::m2", payload: payload({ topics: ["t1"], entities: ["e1", "e2"] }) }, + ], + } as unknown as QdrantStore; + const app = createApi({ + cfg: { + PORT: 0, + HOST: "0.0.0.0", + REQUIRE_API_KEY: false, + MAX_BODY_BYTES: 10_000, + MAX_UPDATE_BODY_BYTES: 10_000, + } as unknown as DeepMemoryServerConfig, + retriever: { + retrieve: async () => ({ entities: [], topics: [], memories: [], context: "" }), + } as unknown as DeepMemoryRetriever, + updater: { + update: async () => ({ status: "processed", memories_added: 0, memories_filtered: 0 }), + } as unknown as DeepMemoryUpdater, + qdrant, + neo4j: {} as unknown as Neo4jStore, + queue: { + enqueue: async () => ({ status: "queued", key: "k", transcriptHash: "h" }), + runNow: async () => ({ status: "processed", memories_added: 0, memories_filtered: 0 }), + cancelBySession: async () => 0, + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + listFailed: async () => [], + retryFailed: async () => ({ status: "not_found" }), + exportFailed: async () => ({ mode: "empty" }), + retryFailedByKey: async () => ({ status: "ok", matched: 0, retried: 0 }), + } as unknown as DurableUpdateQueue, + forgetQueue: { + stats: () => ({ pendingApprox: 0, active: 0, inflightKeys: 0 }), + enqueue: async () => ({ status: "queued", key: "k", taskId: "t" }), + listFailed: async () => [], + exportFailed: async () => ({ mode: "empty" }), + retryFailed: async () => ({ status: "not_found" }), + retryFailedByKey: async () => ({ status: "ok", matched: 0, retried: 0 }), + } as unknown as DurableForgetQueue, + }); + + const res = await app.request("/session/inspect", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ namespace: "default", session_id: "s1", limit: 100 }), + }); + expect(res.status).toBe(200); + const json = (await res.json()) as Record; + expect(json.session_id).toBe("s1"); + const topics = json.topics as Array<{ name: string; frequency: number }>; + const entities = json.entities as Array<{ name: string; frequency: number }>; + expect(topics.find((t) => t.name === "t1")?.frequency).toBe(2); + expect(entities.find((e) => e.name === "e1")?.frequency).toBe(2); + }); +}); diff --git a/packages/deep-memory-server/src/api.ts b/packages/deep-memory-server/src/api.ts new file mode 100644 index 0000000000000..403e373c0031c --- /dev/null +++ b/packages/deep-memory-server/src/api.ts @@ -0,0 +1,1122 @@ +import crypto from "node:crypto"; +import { readFileSync } from "node:fs"; +import { Hono } from "hono"; +import type { Logger } from "pino"; +import { z } from "zod"; +import { extractHintsFromText } from "./analyzer.js"; +import { appendAuditLog } from "./audit-log.js"; +import { createAuthz } from "./authz.js"; +import { enforceBodySize, readJsonWithLimit } from "./body-limit.js"; +import type { DeepMemoryServerConfig } from "./config.js"; +import type { DurableForgetQueue } from "./durable-forget-queue.js"; +import type { DurableUpdateQueue } from "./durable-update-queue.js"; +import type { DeepMemoryMetrics } from "./metrics.js"; +import type { Neo4jStore } from "./neo4j.js"; +import type { QdrantStore } from "./qdrant.js"; +import { FixedWindowRateLimiter } from "./rate-limit.js"; +import type { DeepMemoryRetriever } from "./retriever.js"; +import { DEEPMEM_SCHEMA_VERSION } from "./schema.js"; +import type { + InspectSessionResponse, + RetrieveContextResponse, + UpdateMemoryIndexResponse, +} from "./types.js"; +import type { DeepMemoryUpdater } from "./updater.js"; +import { stableHash } from "./utils.js"; + +type PackageJson = { version?: unknown; name?: unknown }; + +let cachedServiceVersion: string | null = null; + +function getServiceVersion(): string { + if (cachedServiceVersion !== null) { + return cachedServiceVersion; + } + try { + const raw = readFileSync(new URL("../package.json", import.meta.url), "utf-8"); + const parsed = JSON.parse(raw) as unknown; + if (typeof parsed === "object" && parsed) { + const pkg = parsed as PackageJson; + if (typeof pkg.version === "string" && pkg.version.trim()) { + cachedServiceVersion = pkg.version.trim(); + return cachedServiceVersion; + } + } + } catch { + // ignore: best-effort only (e.g. tests or packaged runtime) + } + cachedServiceVersion = "unknown"; + return cachedServiceVersion; +} + +function getBuildInfo(): { sha?: string; ref?: string; time?: string } | undefined { + const sha = (process.env.GIT_SHA ?? process.env.BUILD_SHA ?? "").trim() || undefined; + const ref = (process.env.GIT_REF ?? process.env.BUILD_REF ?? "").trim() || undefined; + const time = (process.env.BUILD_TIME ?? process.env.BUILD_DATE ?? "").trim() || undefined; + if (!sha && !ref && !time) { + return undefined; + } + return { sha, ref, time }; +} + +const RetrieveSchema = z.object({ + namespace: z.string().optional(), + user_input: z.string(), + session_id: z.string(), + max_memories: z.number().int().positive().optional(), +}); + +const UpdateSchema = z.object({ + namespace: z.string().optional(), + session_id: z.string(), + messages: z.array(z.unknown()), + async: z.boolean().optional(), + return_memory_ids: z.boolean().optional(), + max_memory_ids: z.number().int().positive().max(1000).optional(), +}); + +const ForgetSchema = z + .object({ + namespace: z.string().optional(), + memory_ids: z.array(z.string()).optional(), + session_id: z.string().optional(), + dry_run: z.boolean().optional(), + async: z.boolean().optional(), + }) + .strict() + .superRefine((val, ctx) => { + if ((!val.memory_ids || val.memory_ids.length === 0) && !val.session_id) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "must provide memory_ids or session_id", + }); + } + }); + +const InspectSessionSchema = z.object({ + namespace: z.string().optional(), + session_id: z.string(), + limit: z.number().int().positive().max(1000).optional(), + include_content: z.boolean().optional(), +}); + +export function createApi(params: { + cfg: DeepMemoryServerConfig; + log?: Logger; + retriever: DeepMemoryRetriever; + updater: DeepMemoryUpdater; + qdrant: QdrantStore; + neo4j: Neo4jStore; + queue: DurableUpdateQueue; + forgetQueue: DurableForgetQueue; + metrics?: DeepMemoryMetrics; +}) { + const app = new Hono(); + const authz = createAuthz(params.cfg); + const limiter = new FixedWindowRateLimiter({ windowMs: params.cfg.RATE_LIMIT_WINDOW_MS }); + const lastUpdateAtByKey = new Map(); + const activeRetrieveByNamespace = new Map(); + const updateDisabledNamespaces = new Set( + (params.cfg.UPDATE_DISABLED_NAMESPACES ?? "") + .split(/[,\s]+/g) + .map((s) => s.trim()) + .filter(Boolean), + ); + const serviceVersion = getServiceVersion(); + const build = getBuildInfo(); + + const withTimeout = async ( + name: string, + ms: number, + fn: () => Promise, + ): Promise<{ ok: true; value: T } | { ok: false; error: string }> => { + let timer: NodeJS.Timeout | undefined; + try { + const value = await new Promise((resolve, reject) => { + timer = setTimeout(() => reject(new Error(`${name} timeout after ${ms}ms`)), ms); + fn().then(resolve, reject); + }); + return { ok: true, value }; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : String(err) }; + } finally { + if (timer) { + clearTimeout(timer); + } + } + }; + + app.use("*", enforceBodySize(params.cfg)); + + // Request logger: stable request id + latency + auth keyId (never log raw API key). + app.use("*", async (c, next) => { + const incoming = c.req.header("x-request-id")?.trim() || ""; + const requestId = + incoming.length > 0 && incoming.length <= 128 ? incoming : crypto.randomUUID(); + c.header("x-request-id", requestId); + const start = performance.now(); + try { + await next(); + } catch (err) { + params.log?.error( + { + requestId, + method: c.req.method, + path: c.req.path, + keyId: authz.getAuth(c)?.keyId, + err: err instanceof Error ? err.message : String(err), + }, + "request error", + ); + throw err; + } finally { + const status = c.res?.status ?? 500; + const ms = Math.max(0, Math.round(performance.now() - start)); + params.log?.info( + { + requestId, + method: c.req.method, + path: c.req.path, + status, + ms, + keyId: authz.getAuth(c)?.keyId, + }, + "request", + ); + } + }); + // Permission matrix: + // - retrieve_context: read + // - update_memory_index: write + // - forget: admin + // - queue/*: admin + app.use("*", authz.requirePrefix("/queue", "admin")); + app.use("/forget", authz.requireRole("admin")); + app.use("/update_memory_index", authz.requireRole("write")); + if (authz.required) { + app.use("/retrieve_context", authz.requireRole("read")); + app.use("/session/inspect", authz.requireRole("read")); + } + + const checkRateLimit = (c: import("hono").Context, route: string, limit: number) => { + if (!params.cfg.RATE_LIMIT_ENABLED) { + return { ok: true as const }; + } + if (limit <= 0) { + return { ok: true as const }; + } + const keyId = authz.getAuth(c)?.keyId ?? "anon"; + const out = limiter.take({ key: `${keyId}::${route}`, limit }); + if (!out.ok) { + const retryAfterSeconds = Math.max(1, Math.ceil((out.resetAtMs - Date.now()) / 1000)); + c.header("retry-after", String(retryAfterSeconds)); + return { ok: false as const }; + } + return { ok: true as const }; + }; + + // Metrics endpoint: protected if auth is required (recommended). + if (params.metrics) { + if (authz.required) { + app.use("/metrics", authz.requireRole("admin")); + } + app.get("/metrics", async (c) => { + if (!authz.required && !params.cfg.ALLOW_UNAUTHENTICATED_METRICS) { + return c.text("not found", 404); + } + const stats = params.queue.stats(); + params.metrics!.queuePending.set(stats.pendingApprox); + params.metrics!.queueActive.set(stats.active); + params.metrics!.queueInflightKeys.set(stats.inflightKeys); + c.header("content-type", params.metrics!.registry.contentType); + return c.text(await params.metrics!.registry.metrics()); + }); + } + + // Request-level metrics: best-effort, route key uses path (stable). + app.use("*", async (c, next) => { + const start = performance.now(); + try { + await next(); + } finally { + const metrics = params.metrics; + if (metrics) { + const route = c.req.path; + const method = c.req.method; + const status = String(c.res.status ?? 200); + const seconds = Math.max(0, (performance.now() - start) / 1000); + metrics.httpRequestsTotal.labels(route, method, status).inc(); + metrics.httpRequestDurationSeconds.labels(route, method, status).observe(seconds); + } + } + }); + + const buildHealthBody = (details: boolean) => { + const queueStats = params.queue.stats(); + return { + ok: true, + service: { + name: "deep-memory-server", + version: serviceVersion, + build: details ? build : undefined, + }, + runtime: { + node: process.version, + pid: process.pid, + platform: process.platform, + arch: process.arch, + uptimeSec: Math.max(0, Math.floor(process.uptime())), + }, + build: { + sha: params.cfg.BUILD_SHA?.trim() || undefined, + time: params.cfg.BUILD_TIME?.trim() || undefined, + }, + auth: { + required: authz.required, + }, + deps: details + ? { + qdrant: { + url: params.cfg.QDRANT_URL, + collection: params.cfg.QDRANT_COLLECTION, + vectorDims: params.cfg.VECTOR_DIMS, + }, + neo4j: { + uri: params.cfg.NEO4J_URI, + user: params.cfg.NEO4J_USER, + }, + } + : undefined, + queue: { + ...queueStats, + dir: params.cfg.QUEUE_DIR, + keepDone: params.cfg.QUEUE_KEEP_DONE, + retentionDays: params.cfg.QUEUE_RETENTION_DAYS, + maxAttempts: params.cfg.QUEUE_MAX_ATTEMPTS, + updateConcurrency: params.cfg.UPDATE_CONCURRENCY, + }, + guardrails: { + rateLimit: { + enabled: params.cfg.RATE_LIMIT_ENABLED, + windowMs: params.cfg.RATE_LIMIT_WINDOW_MS, + retrievePerWindow: params.cfg.RATE_LIMIT_RETRIEVE_PER_WINDOW, + updatePerWindow: params.cfg.RATE_LIMIT_UPDATE_PER_WINDOW, + forgetPerWindow: params.cfg.RATE_LIMIT_FORGET_PER_WINDOW, + queueAdminPerWindow: params.cfg.RATE_LIMIT_QUEUE_ADMIN_PER_WINDOW, + }, + updateBacklog: { + rejectPending: params.cfg.UPDATE_BACKLOG_REJECT_PENDING, + retryAfterSeconds: params.cfg.UPDATE_BACKLOG_RETRY_AFTER_SECONDS, + delayPending: params.cfg.UPDATE_BACKLOG_DELAY_PENDING, + delaySeconds: params.cfg.UPDATE_BACKLOG_DELAY_SECONDS, + readOnlyPending: params.cfg.UPDATE_BACKLOG_READ_ONLY_PENDING, + }, + updateIngest: { + disabledNamespaces: Array.from(updateDisabledNamespaces.values()), + minIntervalMs: params.cfg.UPDATE_MIN_INTERVAL_MS, + sampleRate: params.cfg.UPDATE_SAMPLE_RATE, + }, + namespaceConcurrency: { + retrieve: params.cfg.NAMESPACE_RETRIEVE_CONCURRENCY, + update: params.cfg.NAMESPACE_UPDATE_CONCURRENCY, + }, + retrieveDegrade: { + relatedPending: params.cfg.RETRIEVE_DEGRADE_RELATED_PENDING, + }, + sensitiveFilter: { + enabled: params.cfg.SENSITIVE_FILTER_ENABLED, + rulesetVersion: params.cfg.SENSITIVE_RULESET_VERSION, + customAllow: Boolean(params.cfg.SENSITIVE_ALLOW_REGEX_JSON), + customDeny: Boolean(params.cfg.SENSITIVE_DENY_REGEX_JSON), + }, + }, + schema: details + ? { + expectedVersion: DEEPMEM_SCHEMA_VERSION, + mode: params.cfg.MIGRATIONS_MODE, + } + : undefined, + now: new Date().toISOString(), + }; + }; + + app.get("/health", (c) => c.json(buildHealthBody(false))); + + // Detailed health is useful for ops; protect it when API keys are required. + if (authz.required) { + app.use("/health/details", authz.requireRole("admin")); + } + app.get("/health/details", async (c) => { + const timeoutMs = 1500; + const qdrant = await withTimeout("qdrant_schema", timeoutMs, async () => + params.qdrant.schemaStatus({ + mode: params.cfg.MIGRATIONS_MODE, + expectedVersion: DEEPMEM_SCHEMA_VERSION, + }), + ); + const neo4j = await withTimeout("neo4j_schema", timeoutMs, async () => + params.neo4j.schemaStatus({ + mode: params.cfg.MIGRATIONS_MODE, + expectedVersion: DEEPMEM_SCHEMA_VERSION, + }), + ); + return c.json({ + ...buildHealthBody(true), + schema: { + expectedVersion: DEEPMEM_SCHEMA_VERSION, + mode: params.cfg.MIGRATIONS_MODE, + qdrant: qdrant.ok ? qdrant.value : { ok: false, error: qdrant.error }, + neo4j: neo4j.ok ? neo4j.value : { ok: false, error: neo4j.error }, + }, + }); + }); + + // Readiness probe: verifies dependencies are reachable within a tight timeout. + app.get("/readyz", async (c) => { + const timeoutMs = 1500; + const qdrant = await withTimeout("qdrant", timeoutMs, async () => params.qdrant.healthCheck()); + const neo4j = await withTimeout("neo4j", timeoutMs, async () => params.neo4j.healthCheck()); + const queue = params.queue.stats(); + + const qdrantResult = qdrant.ok ? qdrant.value : { ok: false as const, error: qdrant.error }; + const neo4jResult = neo4j.ok ? neo4j.value : { ok: false as const, error: neo4j.error }; + const ok = qdrantResult.ok && neo4jResult.ok; + + return c.json( + { + ok, + qdrant: qdrantResult, + neo4j: neo4jResult, + queue, + }, + ok ? 200 : 503, + ); + }); + + app.post("/retrieve_context", async (c) => { + const rl = checkRateLimit(c, "/retrieve_context", params.cfg.RATE_LIMIT_RETRIEVE_PER_WINDOW); + if (!rl.ok) { + return c.json({ error: "rate_limited" }, 429); + } + const body = await readJsonWithLimit>(c, { + limitBytes: params.cfg.MAX_BODY_BYTES, + fallback: {}, + }); + if (typeof body === "object" && body && "error" in body) { + return c.json(body, body.error === "payload_too_large" ? 413 : 400); + } + const parsed = RetrieveSchema.safeParse(body); + if (!parsed.success) { + return c.json({ error: "invalid_request", details: parsed.error.issues }, 400); + } + const req = parsed.data; + const namespace = req.namespace?.trim() || "default"; + const nsCheck = authz.assertNamespace(c, namespace); + if (!nsCheck.ok) { + return c.json(nsCheck.body, nsCheck.status); + } + + const nsRetrieveLimit = params.cfg.NAMESPACE_RETRIEVE_CONCURRENCY; + if (nsRetrieveLimit > 0) { + const active = activeRetrieveByNamespace.get(namespace) ?? 0; + if (active >= nsRetrieveLimit) { + c.header("retry-after", "1"); + return c.json( + { + error: "namespace_overloaded", + namespace, + active, + limit: nsRetrieveLimit, + retryAfterSeconds: 1, + }, + 503, + ); + } + activeRetrieveByNamespace.set(namespace, active + 1); + } + const maxMemories = req.max_memories ?? 10; + const degradeRelatedPending = params.cfg.RETRIEVE_DEGRADE_RELATED_PENDING; + const pendingApprox = params.queue.stats().pendingApprox; + const degradeRelated = degradeRelatedPending > 0 && pendingApprox >= degradeRelatedPending; + const hints = degradeRelated + ? { entities: [] as string[], topics: [] as string[] } + : extractHintsFromText(req.user_input); + try { + const out: RetrieveContextResponse = await params.retriever.retrieve({ + namespace, + userInput: req.user_input, + sessionId: req.session_id, + maxMemories, + entities: hints.entities, + topics: hints.topics, + }); + params.metrics?.retrieveReturnedMemoriesTotal + .labels("200") + .inc(Array.isArray(out.memories) ? out.memories.length : 0); + return c.json(out); + } finally { + if (nsRetrieveLimit > 0) { + const next = (activeRetrieveByNamespace.get(namespace) ?? 1) - 1; + if (next <= 0) { + activeRetrieveByNamespace.delete(namespace); + } else { + activeRetrieveByNamespace.set(namespace, next); + } + } + } + }); + + app.post("/session/inspect", async (c) => { + const rl = checkRateLimit(c, "/session/inspect", params.cfg.RATE_LIMIT_RETRIEVE_PER_WINDOW); + if (!rl.ok) { + return c.json({ error: "rate_limited" }, 429); + } + const body = await readJsonWithLimit>(c, { + limitBytes: params.cfg.MAX_BODY_BYTES, + fallback: {}, + }); + if (typeof body === "object" && body && "error" in body) { + return c.json(body, body.error === "payload_too_large" ? 413 : 400); + } + const parsed = InspectSessionSchema.safeParse(body); + if (!parsed.success) { + return c.json({ error: "invalid_request", details: parsed.error.issues }, 400); + } + const req = parsed.data; + const namespace = req.namespace?.trim() || "default"; + const nsCheck = authz.assertNamespace(c, namespace); + if (!nsCheck.ok) { + return c.json(nsCheck.body, nsCheck.status); + } + const limit = Math.max(1, Math.min(200, Math.trunc(req.limit ?? 100))); + const includeContent = req.include_content ?? false; + + const items = await params.qdrant.listMemoriesBySession({ + namespace, + sessionId: req.session_id, + limit, + }); + const topicsFreq = new Map(); + const entitiesFreq = new Map(); + const memories = items.map((it) => { + const p = it.payload; + const topics = Array.isArray(p?.topics) ? p.topics : []; + const entities = Array.isArray(p?.entities) ? p.entities : []; + for (const t of topics) { + topicsFreq.set(t, (topicsFreq.get(t) ?? 0) + 1); + } + for (const e of entities) { + entitiesFreq.set(e, (entitiesFreq.get(e) ?? 0) + 1); + } + const contentRaw = typeof p?.content === "string" ? p.content : ""; + const content = includeContent ? contentRaw.slice(0, 500) : undefined; + return { + id: it.id, + importance: typeof p?.importance === "number" ? p.importance : undefined, + created_at: typeof p?.created_at === "string" ? p.created_at : undefined, + content, + topics: topics.length > 0 ? topics.slice(0, 10) : undefined, + entities: entities.length > 0 ? entities.slice(0, 10) : undefined, + }; + }); + const topics = Array.from(topicsFreq.entries()) + .toSorted((a, b) => b[1] - a[1]) + .slice(0, 20) + .map(([name, frequency]) => ({ name, frequency })); + const entities = Array.from(entitiesFreq.entries()) + .toSorted((a, b) => b[1] - a[1]) + .slice(0, 20) + .map(([name, frequency]) => ({ name, frequency })); + + const summary = includeContent + ? memories + .toSorted((a, b) => (b.importance ?? 0) - (a.importance ?? 0)) + .slice(0, 3) + .map((m) => (m.content ?? "").trim()) + .filter(Boolean) + .join("\n\n") + .slice(0, 1200) || undefined + : undefined; + + const resp: InspectSessionResponse = { + namespace, + session_id: req.session_id, + totals: { + memories: items.length, + }, + topics, + entities, + memories, + summary, + }; + return c.json(resp); + }); + + app.post("/update_memory_index", async (c) => { + const rl = checkRateLimit(c, "/update_memory_index", params.cfg.RATE_LIMIT_UPDATE_PER_WINDOW); + if (!rl.ok) { + return c.json({ error: "rate_limited" }, 429); + } + const body = await readJsonWithLimit>(c, { + limitBytes: params.cfg.MAX_UPDATE_BODY_BYTES, + fallback: {}, + }); + if (typeof body === "object" && body && "error" in body) { + return c.json(body, body.error === "payload_too_large" ? 413 : 400); + } + const parsed = UpdateSchema.safeParse(body); + if (!parsed.success) { + return c.json({ error: "invalid_request", details: parsed.error.issues }, 400); + } + const req = parsed.data; + const namespace = req.namespace?.trim() || "default"; + const nsCheck = authz.assertNamespace(c, namespace); + if (!nsCheck.ok) { + return c.json(nsCheck.body, nsCheck.status); + } + + if (updateDisabledNamespaces.has(namespace)) { + return c.json({ + status: "skipped", + memories_added: 0, + memories_filtered: 0, + error: "namespace_write_disabled", + }); + } + + const sampleRate = params.cfg.UPDATE_SAMPLE_RATE; + if (sampleRate < 1) { + const msgCount = Array.isArray(req.messages) ? req.messages.length : 0; + const h = stableHash(`${namespace}::${req.session_id}::${msgCount}`); + const bucket = Number.parseInt(h.slice(0, 8), 16); + const p = (bucket % 10_000) / 10_000; + if (p >= sampleRate) { + return c.json({ + status: "skipped", + memories_added: 0, + memories_filtered: 0, + error: "sampled_out", + }); + } + } + + const minIntervalMs = params.cfg.UPDATE_MIN_INTERVAL_MS; + if (minIntervalMs > 0) { + const key = `${namespace}::${req.session_id}`; + const last = lastUpdateAtByKey.get(key) ?? 0; + const now = Date.now(); + const waitMs = last > 0 ? minIntervalMs - (now - last) : 0; + if (waitMs > 0) { + c.header("retry-after", String(Math.max(1, Math.ceil(waitMs / 1000)))); + return c.json({ + status: "skipped", + memories_added: 0, + memories_filtered: 0, + error: "throttled", + }); + } + lastUpdateAtByKey.set(key, now); + } + const runAsync = req.async ?? true; + + if (runAsync) { + const stats = params.queue.stats(); + const readOnlyPending = params.cfg.UPDATE_BACKLOG_READ_ONLY_PENDING; + if (readOnlyPending > 0 && stats.pendingApprox >= readOnlyPending) { + const retryAfter = params.cfg.UPDATE_BACKLOG_RETRY_AFTER_SECONDS; + c.header("retry-after", String(retryAfter)); + return c.json( + { + error: "degraded_read_only", + pendingApprox: stats.pendingApprox, + retryAfterSeconds: retryAfter, + }, + 503, + ); + } + const rejectPending = params.cfg.UPDATE_BACKLOG_REJECT_PENDING; + if (rejectPending > 0) { + if (stats.pendingApprox >= rejectPending) { + const retryAfter = params.cfg.UPDATE_BACKLOG_RETRY_AFTER_SECONDS; + c.header("retry-after", String(retryAfter)); + return c.json( + { + error: "queue_overloaded", + pendingApprox: stats.pendingApprox, + retryAfterSeconds: retryAfter, + }, + 503, + ); + } + } + const delayPending = params.cfg.UPDATE_BACKLOG_DELAY_PENDING; + const delaySeconds = params.cfg.UPDATE_BACKLOG_DELAY_SECONDS; + const notBeforeMs = + delayPending > 0 && delaySeconds > 0 && stats.pendingApprox >= delayPending + ? Date.now() + delaySeconds * 1000 + : undefined; + void params.queue.enqueue({ + namespace, + sessionId: req.session_id, + messages: req.messages, + notBeforeMs, + }); + const resp: UpdateMemoryIndexResponse = { + status: "queued", + memories_added: 0, + memories_filtered: 0, + }; + if (notBeforeMs) { + (resp as unknown as Record).degraded = { + mode: "delayed", + notBeforeMs, + delaySeconds, + }; + } + return c.json(resp); + } + + const result = await params.queue.runNow({ + namespace, + sessionId: req.session_id, + messages: req.messages, + returnMemoryIds: req.return_memory_ids + ? { + max: Math.max(1, Math.min(1000, Math.trunc(req.max_memory_ids ?? 200))), + } + : undefined, + }); + params.metrics?.updateMemoriesAddedTotal.labels("200").inc(result.memories_added ?? 0); + params.metrics?.updateMemoriesFilteredTotal.labels("200").inc(result.memories_filtered ?? 0); + return c.json(result); + }); + + // Minimal “forget” API: delete by explicit ids or by session. + // This is intentionally best-effort and should be protected at the network layer. + app.post("/forget", async (c) => { + const rl = checkRateLimit(c, "/forget", params.cfg.RATE_LIMIT_FORGET_PER_WINDOW); + if (!rl.ok) { + return c.json({ error: "rate_limited" }, 429); + } + const body = await readJsonWithLimit>(c, { + limitBytes: params.cfg.MAX_BODY_BYTES, + fallback: {}, + }); + if (typeof body === "object" && body && "error" in body) { + return c.json(body, body.error === "payload_too_large" ? 413 : 400); + } + const parsed = ForgetSchema.safeParse(body); + if (!parsed.success) { + return c.json({ error: "invalid_request", details: parsed.error.issues }, 400); + } + const req = parsed.data; + const namespace = req.namespace?.trim() || "default"; + const nsCheck = authz.assertNamespace(c, namespace); + if (!nsCheck.ok) { + return c.json(nsCheck.body, nsCheck.status); + } + const dryRun = req.dry_run ?? false; + const runAsync = req.async ?? false; + + const ids = (req.memory_ids ?? []).map((id) => id.trim()).filter(Boolean); + const normalizedIds = ids.map((id) => (id.includes("::") ? id : `${namespace}::${id}`)); + const requestId = c.res.headers.get("x-request-id") ?? undefined; + + if (dryRun) { + await appendAuditLog(params.cfg, { + action: "forget", + namespace, + requestId, + dryRun: true, + sessionId: req.session_id ?? undefined, + memoryIdsCount: normalizedIds.length, + requester: { + ip: c.req.header("x-forwarded-for") ?? c.req.header("x-real-ip") ?? undefined, + userAgent: c.req.header("user-agent") ?? undefined, + keyId: authz.getAuth(c)?.keyId, + }, + }).catch(() => {}); + return c.json({ + status: "dry_run", + namespace, + request_id: requestId, + delete_ids: normalizedIds.length, + delete_session: req.session_id ? 1 : 0, + }); + } + + if (runAsync) { + const out = await params.forgetQueue.enqueue({ + namespace, + sessionId: req.session_id ?? undefined, + memoryIds: normalizedIds, + }); + await appendAuditLog(params.cfg, { + action: "forget", + namespace, + requestId, + dryRun: false, + sessionId: req.session_id ?? undefined, + memoryIdsCount: normalizedIds.length, + deletedReported: 0, + results: { + queue: { ok: true, cancelled: 0 }, + }, + requester: { + ip: c.req.header("x-forwarded-for") ?? c.req.header("x-real-ip") ?? undefined, + userAgent: c.req.header("user-agent") ?? undefined, + keyId: authz.getAuth(c)?.keyId, + }, + }).catch(() => {}); + return c.json({ + status: "queued", + namespace, + request_id: requestId, + key: out.key, + task_id: out.taskId, + delete_ids: normalizedIds.length, + delete_session: req.session_id ? 1 : 0, + }); + } + + let deleted = 0; + const results: NonNullable< + Parameters[1] & { action: "forget" } + >["results"] = { + qdrant: {}, + neo4j: {}, + queue: { ok: true, cancelled: 0 }, + }; + if (req.session_id) { + // Best-effort dual delete. + try { + await params.qdrant.deleteBySession({ namespace, sessionId: req.session_id }); + results.qdrant!.bySession = { ok: true }; + } catch { + results.qdrant!.bySession = { ok: false, error: "qdrant deleteBySession failed" }; + } + try { + const d = await params.neo4j.deleteMemoriesBySession({ + namespace, + sessionId: req.session_id, + }); + deleted += d; + results.neo4j!.bySession = { ok: true, deleted: d }; + } catch { + results.neo4j!.bySession = { ok: false, error: "neo4j deleteMemoriesBySession failed" }; + } + try { + const cancelled = await params.queue.cancelBySession({ + namespace, + sessionId: req.session_id, + }); + results.queue = { ok: true, cancelled }; + } catch { + results.queue = { ok: false, cancelled: 0, error: "queue cancelBySession failed" }; + } + } + if (normalizedIds.length > 0) { + try { + const d = await params.qdrant.deleteByIds({ ids: normalizedIds }); + results.qdrant!.byIds = { ok: true, deleted: d }; + } catch { + results.qdrant!.byIds = { ok: false, error: "qdrant deleteByIds failed" }; + } + try { + const d = await params.neo4j.deleteMemoriesByIds({ namespace, ids: normalizedIds }); + deleted += d; + results.neo4j!.byIds = { ok: true, deleted: d }; + } catch { + results.neo4j!.byIds = { ok: false, error: "neo4j deleteMemoriesByIds failed" }; + } + } + await appendAuditLog(params.cfg, { + action: "forget", + namespace, + requestId, + dryRun: false, + sessionId: req.session_id ?? undefined, + memoryIdsCount: normalizedIds.length, + deletedReported: deleted, + results, + requester: { + ip: c.req.header("x-forwarded-for") ?? c.req.header("x-real-ip") ?? undefined, + userAgent: c.req.header("user-agent") ?? undefined, + keyId: authz.getAuth(c)?.keyId, + }, + }).catch(() => {}); + params.metrics?.forgetDeletedTotal.labels("200").inc(deleted); + return c.json({ status: "processed", namespace, request_id: requestId, deleted, results }); + }); + + app.get("/queue/stats", (c) => { + const rl = checkRateLimit(c, "/queue", params.cfg.RATE_LIMIT_QUEUE_ADMIN_PER_WINDOW); + if (!rl.ok) { + return c.json({ error: "rate_limited" }, 429); + } + return c.json({ ok: true, ...params.queue.stats() }); + }); + + app.get("/queue/forget/stats", (c) => { + const rl = checkRateLimit(c, "/queue", params.cfg.RATE_LIMIT_QUEUE_ADMIN_PER_WINDOW); + if (!rl.ok) { + return c.json({ error: "rate_limited" }, 429); + } + return c.json({ ok: true, ...params.forgetQueue.stats() }); + }); + + app.get("/queue/forget/failed", async (c) => { + const rl = checkRateLimit(c, "/queue", params.cfg.RATE_LIMIT_QUEUE_ADMIN_PER_WINDOW); + if (!rl.ok) { + return c.json({ error: "rate_limited" }, 429); + } + const limitRaw = c.req.query("limit"); + const limit = Math.max(1, Math.min(200, Number(limitRaw ?? 50) || 50)); + const items = await params.forgetQueue.listFailed({ limit }); + return c.json({ ok: true, items }); + }); + + app.get("/queue/forget/failed/export", async (c) => { + const rl = checkRateLimit(c, "/queue", params.cfg.RATE_LIMIT_QUEUE_ADMIN_PER_WINDOW); + if (!rl.ok) { + return c.json({ error: "rate_limited" }, 429); + } + const file = c.req.query("file") ?? undefined; + const key = c.req.query("key") ?? undefined; + const limitRaw = c.req.query("limit"); + const limit = Math.max(1, Math.min(200, Number(limitRaw ?? 50) || 50)); + + const ns = key ? authz.extractNamespaceFromKey(key) : null; + if (ns) { + const nsCheck = authz.assertNamespace(c, ns); + if (!nsCheck.ok) { + return c.json(nsCheck.body, nsCheck.status); + } + } + const out = await params.forgetQueue.exportFailed({ file, key, limit }); + await appendAuditLog(params.cfg, { + action: "queue_failed_export", + queueKind: "forget", + file: file?.trim() || undefined, + key: key?.trim() || undefined, + limit, + requester: { + ip: c.req.header("x-forwarded-for") ?? c.req.header("x-real-ip") ?? undefined, + userAgent: c.req.header("user-agent") ?? undefined, + keyId: authz.getAuth(c)?.keyId, + }, + }).catch(() => {}); + return c.json({ ok: true, ...out }); + }); + + app.post("/queue/forget/failed/retry", async (c) => { + const rl = checkRateLimit(c, "/queue", params.cfg.RATE_LIMIT_QUEUE_ADMIN_PER_WINDOW); + if (!rl.ok) { + return c.json({ error: "rate_limited" }, 429); + } + const body = await readJsonWithLimit>(c, { + limitBytes: params.cfg.MAX_BODY_BYTES, + fallback: {}, + }); + if (typeof body === "object" && body && "error" in body) { + return c.json(body, body.error === "payload_too_large" ? 413 : 400); + } + const file = typeof body.file === "string" ? body.file : ""; + const key = typeof body.key === "string" ? body.key : ""; + const dryRun = body.dry_run === true || body.dry_run === "true"; + const limitRaw = + typeof body.limit === "number" + ? body.limit + : typeof body.limit === "string" + ? Number(body.limit) + : 50; + const limit = Math.max(1, Math.min(200, Number(limitRaw) || 50)); + if (file) { + if (dryRun) { + await appendAuditLog(params.cfg, { + action: "queue_failed_retry", + queueKind: "forget", + dryRun: true, + file, + requester: { + ip: c.req.header("x-forwarded-for") ?? c.req.header("x-real-ip") ?? undefined, + userAgent: c.req.header("user-agent") ?? undefined, + keyId: authz.getAuth(c)?.keyId, + }, + }).catch(() => {}); + return c.json({ ok: true, mode: "file", status: "dry_run" }); + } + const out = await params.forgetQueue.retryFailed({ file }); + await appendAuditLog(params.cfg, { + action: "queue_failed_retry", + queueKind: "forget", + dryRun: false, + file, + requester: { + ip: c.req.header("x-forwarded-for") ?? c.req.header("x-real-ip") ?? undefined, + userAgent: c.req.header("user-agent") ?? undefined, + keyId: authz.getAuth(c)?.keyId, + }, + }).catch(() => {}); + return c.json({ ok: true, mode: "file", ...out }); + } + if (key) { + const ns = authz.extractNamespaceFromKey(key); + if (ns) { + const nsCheck = authz.assertNamespace(c, ns); + if (!nsCheck.ok) { + return c.json(nsCheck.body, nsCheck.status); + } + } + const out = await params.forgetQueue.retryFailedByKey({ key, limit, dryRun }); + await appendAuditLog(params.cfg, { + action: "queue_failed_retry", + queueKind: "forget", + dryRun, + key, + limit, + retried: out.retried, + requester: { + ip: c.req.header("x-forwarded-for") ?? c.req.header("x-real-ip") ?? undefined, + userAgent: c.req.header("user-agent") ?? undefined, + keyId: authz.getAuth(c)?.keyId, + }, + }).catch(() => {}); + return c.json({ ok: true, mode: "key", ...out }); + } + return c.json({ error: "invalid_request" }, 400); + }); + + app.get("/queue/failed", async (c) => { + const rl = checkRateLimit(c, "/queue", params.cfg.RATE_LIMIT_QUEUE_ADMIN_PER_WINDOW); + if (!rl.ok) { + return c.json({ error: "rate_limited" }, 429); + } + const limitRaw = c.req.query("limit"); + const limit = Math.max(1, Math.min(200, Number(limitRaw ?? 50) || 50)); + const items = await params.queue.listFailed({ limit }); + return c.json({ ok: true, items }); + }); + + app.get("/queue/failed/export", async (c) => { + const rl = checkRateLimit(c, "/queue", params.cfg.RATE_LIMIT_QUEUE_ADMIN_PER_WINDOW); + if (!rl.ok) { + return c.json({ error: "rate_limited" }, 429); + } + const file = c.req.query("file") ?? undefined; + const key = c.req.query("key") ?? undefined; + const limitRaw = c.req.query("limit"); + const limit = Math.max(1, Math.min(200, Number(limitRaw ?? 50) || 50)); + + const ns = key ? authz.extractNamespaceFromKey(key) : null; + if (ns) { + const nsCheck = authz.assertNamespace(c, ns); + if (!nsCheck.ok) { + return c.json(nsCheck.body, nsCheck.status); + } + } + const out = await params.queue.exportFailed({ file, key, limit }); + await appendAuditLog(params.cfg, { + action: "queue_failed_export", + queueKind: "update", + file: file?.trim() || undefined, + key: key?.trim() || undefined, + limit, + requester: { + ip: c.req.header("x-forwarded-for") ?? c.req.header("x-real-ip") ?? undefined, + userAgent: c.req.header("user-agent") ?? undefined, + keyId: authz.getAuth(c)?.keyId, + }, + }).catch(() => {}); + return c.json({ ok: true, ...out }); + }); + + app.post("/queue/failed/retry", async (c) => { + const rl = checkRateLimit(c, "/queue", params.cfg.RATE_LIMIT_QUEUE_ADMIN_PER_WINDOW); + if (!rl.ok) { + return c.json({ error: "rate_limited" }, 429); + } + const body = await readJsonWithLimit>(c, { + limitBytes: params.cfg.MAX_BODY_BYTES, + fallback: {}, + }); + if (typeof body === "object" && body && "error" in body) { + return c.json(body, body.error === "payload_too_large" ? 413 : 400); + } + const file = typeof body.file === "string" ? body.file : ""; + const key = typeof body.key === "string" ? body.key : ""; + const dryRun = body.dry_run === true || body.dry_run === "true"; + const limitRaw = + typeof body.limit === "number" + ? body.limit + : typeof body.limit === "string" + ? Number(body.limit) + : 50; + const limit = Math.max(1, Math.min(200, Number(limitRaw) || 50)); + if (file) { + // Namespace binding for file retry: peek meta via export first. + const meta = await params.queue.exportFailed({ file, limit: 1 }); + const ns = meta.mode === "file" ? meta.item.namespace : null; + if (ns) { + const nsCheck = authz.assertNamespace(c, ns); + if (!nsCheck.ok) { + return c.json(nsCheck.body, nsCheck.status); + } + } + if (dryRun) { + await appendAuditLog(params.cfg, { + action: "queue_failed_retry", + queueKind: "update", + dryRun: true, + file, + requester: { + ip: c.req.header("x-forwarded-for") ?? c.req.header("x-real-ip") ?? undefined, + userAgent: c.req.header("user-agent") ?? undefined, + keyId: authz.getAuth(c)?.keyId, + }, + }).catch(() => {}); + return c.json({ ok: true, mode: "file", status: "dry_run" }); + } + const out = await params.queue.retryFailed({ file }); + await appendAuditLog(params.cfg, { + action: "queue_failed_retry", + queueKind: "update", + dryRun: false, + file, + requester: { + ip: c.req.header("x-forwarded-for") ?? c.req.header("x-real-ip") ?? undefined, + userAgent: c.req.header("user-agent") ?? undefined, + keyId: authz.getAuth(c)?.keyId, + }, + }).catch(() => {}); + return c.json({ ok: true, mode: "file", ...out }); + } + if (key) { + const ns = authz.extractNamespaceFromKey(key); + if (ns) { + const nsCheck = authz.assertNamespace(c, ns); + if (!nsCheck.ok) { + return c.json(nsCheck.body, nsCheck.status); + } + } + const out = await params.queue.retryFailedByKey({ key, limit, dryRun }); + await appendAuditLog(params.cfg, { + action: "queue_failed_retry", + queueKind: "update", + dryRun, + key, + limit, + retried: out.retried, + requester: { + ip: c.req.header("x-forwarded-for") ?? c.req.header("x-real-ip") ?? undefined, + userAgent: c.req.header("user-agent") ?? undefined, + keyId: authz.getAuth(c)?.keyId, + }, + }).catch(() => {}); + return c.json({ ok: true, mode: "key", ...out }); + } + return c.json({ error: "invalid_request" }, 400); + }); + + return app; +} diff --git a/packages/deep-memory-server/src/audit-log.ts b/packages/deep-memory-server/src/audit-log.ts new file mode 100644 index 0000000000000..a0127a3d27dd6 --- /dev/null +++ b/packages/deep-memory-server/src/audit-log.ts @@ -0,0 +1,96 @@ +import crypto from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import type { DeepMemoryServerConfig } from "./config.js"; + +export type AuditRequester = { + ip?: string; + userAgent?: string; + keyId?: string; +}; + +type QueueKind = "update" | "forget"; + +type ForgetAuditEntry = { + id: string; + ts: string; + action: "forget"; + namespace: string; + requestId?: string; + dryRun: boolean; + sessionId?: string; + memoryIdsCount: number; + deletedReported?: number; + results?: { + qdrant?: { + bySession?: { ok: boolean; error?: string }; + byIds?: { ok: boolean; deleted?: number; error?: string }; + }; + neo4j?: { + bySession?: { ok: boolean; deleted?: number; error?: string }; + byIds?: { ok: boolean; deleted?: number; error?: string }; + }; + queue?: { ok: boolean; cancelled?: number; error?: string }; + }; + requester: AuditRequester; +}; + +type QueueFailedExportAuditEntry = { + id: string; + ts: string; + action: "queue_failed_export"; + /** + * Disambiguates update queue vs forget queue, since both share the same + * admin endpoints shape but act on different durable queues. + */ + queueKind?: QueueKind; + file?: string; + key?: string; + limit?: number; + requester: AuditRequester; +}; + +type QueueFailedRetryAuditEntry = { + id: string; + ts: string; + action: "queue_failed_retry"; + /** + * Disambiguates update queue vs forget queue, since both share the same + * admin endpoints shape but act on different durable queues. + */ + queueKind?: QueueKind; + dryRun: boolean; + file?: string; + key?: string; + limit?: number; + retried?: number; + requester: AuditRequester; +}; + +export type AuditEntry = + | ForgetAuditEntry + | QueueFailedExportAuditEntry + | QueueFailedRetryAuditEntry; + +export type AuditEntryInput = + | Omit + | Omit + | Omit; + +async function ensureParent(filePath: string) { + await fs.mkdir(path.dirname(filePath), { recursive: true }); +} + +export async function appendAuditLog(cfg: DeepMemoryServerConfig, entry: AuditEntryInput) { + const filePath = cfg.AUDIT_LOG_PATH?.trim(); + if (!filePath) { + return; + } + const line: AuditEntry = { + id: crypto.randomUUID(), + ts: new Date().toISOString(), + ...entry, + }; + await ensureParent(filePath); + await fs.appendFile(filePath, `${JSON.stringify(line)}\n`, "utf8"); +} diff --git a/packages/deep-memory-server/src/authz.ts b/packages/deep-memory-server/src/authz.ts new file mode 100644 index 0000000000000..d1e61859f4122 --- /dev/null +++ b/packages/deep-memory-server/src/authz.ts @@ -0,0 +1,182 @@ +import crypto from "node:crypto"; +import type { Context, Next } from "hono"; +import { z } from "zod"; +import type { DeepMemoryServerConfig } from "./config.js"; + +export type AuthRole = "read" | "write" | "admin"; + +export type AuthInfo = { + role: AuthRole; + namespaces: string[]; // ["*"] means all + keyId: string; // short fingerprint for auditing +}; + +type ApiKeyRule = { + key: string; + role: AuthRole; + namespaces?: string[]; +}; + +const ApiKeyRulesSchema = z.array( + z + .object({ + key: z.string().min(1), + role: z.union([z.literal("read"), z.literal("write"), z.literal("admin")]), + namespaces: z.array(z.string()).optional(), + }) + .strict(), +); + +function roleRank(role: AuthRole): number { + return role === "read" ? 1 : role === "write" ? 2 : 3; +} + +function timingSafeEqual(a: string, b: string): boolean { + const ab = Buffer.from(a); + const bb = Buffer.from(b); + const len = Math.max(ab.length, bb.length); + const ap = Buffer.concat([ab, Buffer.alloc(Math.max(0, len - ab.length))]); + const bp = Buffer.concat([bb, Buffer.alloc(Math.max(0, len - bb.length))]); + return crypto.timingSafeEqual(ap, bp) && a.length === b.length; +} + +function keyFingerprint(key: string): string { + return crypto.createHash("sha256").update(key).digest("hex").slice(0, 12); +} + +function parseRules(cfg: DeepMemoryServerConfig): ApiKeyRule[] { + if (cfg.API_KEYS_JSON) { + const raw = cfg.API_KEYS_JSON.trim(); + if (raw) { + const parsed = ApiKeyRulesSchema.safeParse(JSON.parse(raw)); + if (!parsed.success) { + throw new Error("Invalid API_KEYS_JSON"); + } + return parsed.data; + } + } + + const keys: string[] = []; + if (cfg.API_KEY?.trim()) { + keys.push(cfg.API_KEY.trim()); + } + if (cfg.API_KEYS?.trim()) { + for (const part of cfg.API_KEYS.split(",")) { + const k = part.trim(); + if (k) { + keys.push(k); + } + } + } + // Back-compat: if configured via API_KEY/API_KEYS, treat as admin over all namespaces. + return keys.map((k) => ({ key: k, role: "admin", namespaces: ["*"] })); +} + +function normalizeNamespaces(list?: string[]): string[] { + if (!list || list.length === 0) { + return ["*"]; + } + const out = list.map((s) => s.trim()).filter(Boolean); + return out.length === 0 ? ["*"] : out; +} + +function namespaceAllowed(auth: AuthInfo, namespace: string): boolean { + if (auth.namespaces.includes("*")) { + return true; + } + return auth.namespaces.includes(namespace); +} + +export function createAuthz(cfg: DeepMemoryServerConfig) { + const rules = parseRules(cfg).map((r) => ({ + key: r.key, + role: r.role, + namespaces: normalizeNamespaces(r.namespaces), + keyId: keyFingerprint(r.key), + })); + const required = cfg.REQUIRE_API_KEY || rules.length > 0; + if (cfg.REQUIRE_API_KEY && rules.length === 0) { + throw new Error("REQUIRE_API_KEY is set but no API keys are configured"); + } + + const resolveAuth = (provided: string): AuthInfo | null => { + const header = provided.trim(); + if (!header) { + return null; + } + for (const r of rules) { + if (timingSafeEqual(header, r.key)) { + return { role: r.role, namespaces: r.namespaces, keyId: r.keyId }; + } + } + return null; + }; + + const getAuth = (c: Context): AuthInfo | null => { + return (c.get("auth") as AuthInfo | undefined) ?? null; + }; + + const requireRole = (minRole: AuthRole) => { + return async (c: Context, next: Next) => { + if (!required) { + // No keys configured and not required => open access. + return await next(); + } + const header = c.req.header("x-api-key") ?? ""; + const auth = resolveAuth(header); + if (!auth) { + return c.json({ error: "unauthorized" }, 401); + } + if (roleRank(auth.role) < roleRank(minRole)) { + return c.json({ error: "forbidden" }, 403); + } + c.set("auth", auth); + return await next(); + }; + }; + + const requirePrefix = (prefix: string, minRole: AuthRole) => { + const guard = requireRole(minRole); + return async (c: Context, next: Next) => { + if (c.req.path.startsWith(prefix)) { + return await guard(c, next); + } + return await next(); + }; + }; + + const assertNamespace = (c: Context, namespace: string) => { + if (!required) { + return { ok: true as const }; + } + const auth = getAuth(c); + if (!auth) { + return { ok: false as const, status: 401 as const, body: { error: "unauthorized" as const } }; + } + if (!namespaceAllowed(auth, namespace)) { + return { + ok: false as const, + status: 403 as const, + body: { error: "forbidden_namespace" as const }, + }; + } + return { ok: true as const }; + }; + + const extractNamespaceFromKey = (key: string): string | null => { + const idx = key.indexOf("::"); + if (idx <= 0) { + return null; + } + return key.slice(0, idx); + }; + + return { + required, + requireRole, + requirePrefix, + assertNamespace, + extractNamespaceFromKey, + getAuth, + }; +} diff --git a/packages/deep-memory-server/src/body-limit.ts b/packages/deep-memory-server/src/body-limit.ts new file mode 100644 index 0000000000000..a80883616061c --- /dev/null +++ b/packages/deep-memory-server/src/body-limit.ts @@ -0,0 +1,41 @@ +import type { Context, Next } from "hono"; +import type { DeepMemoryServerConfig } from "./config.js"; + +function parseContentLength(raw: string | undefined): number | null { + if (!raw) { + return null; + } + const n = Number(raw); + return Number.isFinite(n) && n >= 0 ? n : null; +} + +export function enforceBodySize(cfg: DeepMemoryServerConfig) { + return async (c: Context, next: Next) => { + const path = c.req.path; + // Conservative: retrieve should be small; update may be large. + const limit = path === "/update_memory_index" ? cfg.MAX_UPDATE_BODY_BYTES : cfg.MAX_BODY_BYTES; + const contentLength = parseContentLength(c.req.header("content-length")); + if (typeof limit === "number" && limit > 0 && contentLength !== null && contentLength > limit) { + return c.json({ error: "payload_too_large", limitBytes: limit }, 413); + } + return await next(); + }; +} + +export async function readJsonWithLimit( + c: Context, + params: { limitBytes: number; fallback: T }, +): Promise { + try { + const buf = Buffer.from(await c.req.arrayBuffer()); + if (buf.length > params.limitBytes) { + return { error: "payload_too_large", limitBytes: params.limitBytes }; + } + if (buf.length === 0) { + return params.fallback; + } + return JSON.parse(buf.toString("utf8")) as T; + } catch { + return { error: "invalid_json" }; + } +} diff --git a/packages/deep-memory-server/src/config.ts b/packages/deep-memory-server/src/config.ts new file mode 100644 index 0000000000000..ce47683d50991 --- /dev/null +++ b/packages/deep-memory-server/src/config.ts @@ -0,0 +1,126 @@ +import { z } from "zod"; +import type { MigrationMode } from "./schema.js"; + +const Bool = z.preprocess((v) => { + if (typeof v === "boolean") { + return v; + } + if (typeof v === "string") { + const s = v.trim().toLowerCase(); + if (s === "" || s === "0" || s === "false" || s === "no" || s === "off") { + return false; + } + if (s === "1" || s === "true" || s === "yes" || s === "on") { + return true; + } + } + return v; +}, z.boolean()); + +const EnvSchema = z.object({ + PORT: z.coerce.number().int().positive().default(8088), + HOST: z.string().default("0.0.0.0"), + API_KEY: z.string().optional(), + API_KEYS: z.string().optional(), + API_KEYS_JSON: z.string().optional(), + BUILD_SHA: z.string().optional(), + BUILD_TIME: z.string().optional(), + REQUIRE_API_KEY: Bool.default(false), + MAX_BODY_BYTES: z.coerce + .number() + .int() + .positive() + .default(256 * 1024), + MAX_UPDATE_BODY_BYTES: z.coerce + .number() + .int() + .positive() + .default(2 * 1024 * 1024), + AUDIT_LOG_PATH: z.string().optional(), + ALLOW_UNAUTHENTICATED_METRICS: Bool.default(false), + RATE_LIMIT_ENABLED: Bool.default(false), + RATE_LIMIT_WINDOW_MS: z.coerce.number().int().positive().default(60_000), + RATE_LIMIT_RETRIEVE_PER_WINDOW: z.coerce.number().int().nonnegative().default(0), + RATE_LIMIT_UPDATE_PER_WINDOW: z.coerce.number().int().nonnegative().default(0), + RATE_LIMIT_FORGET_PER_WINDOW: z.coerce.number().int().nonnegative().default(0), + RATE_LIMIT_QUEUE_ADMIN_PER_WINDOW: z.coerce.number().int().nonnegative().default(0), + UPDATE_BACKLOG_REJECT_PENDING: z.coerce.number().int().nonnegative().default(0), + UPDATE_BACKLOG_RETRY_AFTER_SECONDS: z.coerce.number().int().positive().default(30), + UPDATE_BACKLOG_DELAY_PENDING: z.coerce.number().int().nonnegative().default(0), + UPDATE_BACKLOG_DELAY_SECONDS: z.coerce.number().int().nonnegative().default(0), + UPDATE_BACKLOG_READ_ONLY_PENDING: z.coerce.number().int().nonnegative().default(0), + UPDATE_DISABLED_NAMESPACES: z.string().optional(), + UPDATE_MIN_INTERVAL_MS: z.coerce.number().int().nonnegative().default(0), + UPDATE_SAMPLE_RATE: z.coerce.number().min(0).max(1).default(1), + NAMESPACE_RETRIEVE_CONCURRENCY: z.coerce.number().int().nonnegative().default(0), + NAMESPACE_UPDATE_CONCURRENCY: z.coerce.number().int().nonnegative().default(0), + RETRIEVE_DEGRADE_RELATED_PENDING: z.coerce.number().int().nonnegative().default(0), + MIGRATIONS_MODE: z + .enum(["off", "validate", "apply"]) + .default("apply") satisfies z.ZodType, + MIGRATIONS_STRICT: Bool.default(false), + + // Qdrant + QDRANT_URL: z.string().default("http://qdrant:6333"), + QDRANT_API_KEY: z.string().optional(), + QDRANT_COLLECTION: z.string().default("openclaw_memories"), + VECTOR_DIMS: z.coerce.number().int().positive().default(384), + MIN_SEMANTIC_SCORE: z.coerce.number().min(0).max(1).default(0.6), + SEMANTIC_WEIGHT: z.coerce.number().min(0).max(1).default(0.6), + RELATION_WEIGHT: z.coerce.number().min(0).max(1).default(0.4), + + // Neo4j + NEO4J_URI: z.string().default("bolt://neo4j:7687"), + NEO4J_USER: z.string().default("neo4j"), + NEO4J_PASSWORD: z.string().default("openclaw"), + + // Service behavior + LOG_LEVEL: z.string().default("info"), + RETRIEVE_CACHE_TTL_MS: z.coerce + .number() + .int() + .nonnegative() + .default(5 * 60_000), + RETRIEVE_CACHE_MAX: z.coerce.number().int().positive().default(500), + UPDATE_CONCURRENCY: z.coerce.number().int().positive().default(1), + QUEUE_DIR: z.string().default("./data/queue"), + QUEUE_MAX_ATTEMPTS: z.coerce.number().int().positive().default(10), + QUEUE_RETRY_BASE_MS: z.coerce.number().int().positive().default(2_000), + QUEUE_RETRY_MAX_MS: z.coerce + .number() + .int() + .positive() + .default(5 * 60_000), + QUEUE_KEEP_DONE: Bool.default(true), + QUEUE_RETENTION_DAYS: z.coerce.number().int().positive().default(7), + QUEUE_MAX_TASK_BYTES: z.coerce + .number() + .int() + .positive() + .default(2 * 1024 * 1024), + IMPORTANCE_THRESHOLD: z.coerce.number().min(0).max(1).default(0.5), + MAX_MEMORIES_PER_UPDATE: z.coerce.number().int().positive().default(20), + DEDUPE_SCORE: z.coerce.number().min(0).max(1).default(0.92), + DECAY_HALF_LIFE_DAYS: z.coerce.number().int().positive().default(90), + IMPORTANCE_BOOST: z.coerce.number().min(0).max(2).default(0.3), + FREQUENCY_BOOST: z.coerce.number().min(0).max(2).default(0.2), + RELATED_TOPK: z.coerce.number().int().nonnegative().default(5), + SENSITIVE_FILTER_ENABLED: Bool.default(true), + SENSITIVE_RULESET_VERSION: z.string().default("builtin-v1"), + SENSITIVE_DENY_REGEX_JSON: z.string().optional(), + SENSITIVE_ALLOW_REGEX_JSON: z.string().optional(), + + // Embeddings + EMBEDDING_MODEL: z.string().default("Xenova/bge-small-en-v1.5"), +}); + +export type DeepMemoryServerConfig = z.infer; + +export function loadConfig(env: NodeJS.ProcessEnv = process.env): DeepMemoryServerConfig { + const parsed = EnvSchema.safeParse(env); + if (!parsed.success) { + const issues = parsed.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`).join(", "); + throw new Error(`Invalid deep-memory-server env: ${issues}`); + } + return parsed.data; +} diff --git a/packages/deep-memory-server/src/durable-forget-queue.ts b/packages/deep-memory-server/src/durable-forget-queue.ts new file mode 100644 index 0000000000000..5936d9b85b05e --- /dev/null +++ b/packages/deep-memory-server/src/durable-forget-queue.ts @@ -0,0 +1,522 @@ +import crypto from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import type { Logger } from "pino"; +import type { DurableUpdateQueue } from "./durable-update-queue.js"; +import type { Neo4jStore } from "./neo4j.js"; +import type { QdrantStore } from "./qdrant.js"; + +type ForgetRequest = { + namespace: string; + sessionId?: string; + memoryIds?: string[]; +}; + +type PersistedForgetTask = { + kind: "forget"; + id: string; + key: string; // namespace::(sessionId|ids::) + namespace: string; + sessionId?: string; + memoryIds?: string[]; + createdAt: string; + attempt: number; + nextRunAt: number; + lastError?: string; + result?: { + deletedNeo4j?: number; + qdrant?: { + bySession?: { ok: boolean; error?: string }; + byIds?: { ok: boolean; deleted?: number; error?: string }; + }; + neo4j?: { + bySession?: { ok: boolean; deleted?: number; error?: string }; + byIds?: { ok: boolean; deleted?: number; error?: string }; + }; + queue?: { ok: boolean; cancelled?: number; error?: string }; + }; +}; + +function sleep(ms: number) { + return new Promise((r) => setTimeout(r, ms)); +} + +function stableHash(s: string): string { + return crypto.createHash("sha256").update(s).digest("hex").slice(0, 16); +} + +function backoffMs(params: { baseMs: number; maxMs: number; attempt: number }): number { + const exp = Math.min(20, Math.max(0, params.attempt - 1)); + const raw = Math.min(params.maxMs, params.baseMs * 2 ** exp); + const jitter = Math.floor(Math.random() * Math.min(250, Math.max(10, raw / 10))); + return Math.min(params.maxMs, raw + jitter); +} + +async function ensureDir(dir: string) { + await fs.mkdir(dir, { recursive: true }); +} + +async function atomicWriteJson(filePath: string, value: unknown) { + const dir = path.dirname(filePath); + await ensureDir(dir); + const tmp = `${filePath}.tmp-${crypto.randomUUID()}`; + const data = JSON.stringify(value); + const handle = await fs.open(tmp, "w"); + try { + await handle.writeFile(data, "utf8"); + await handle.sync(); + } finally { + await handle.close(); + } + await fs.rename(tmp, filePath); +} + +async function readJson(filePath: string): Promise { + const raw = await fs.readFile(filePath, "utf8"); + return JSON.parse(raw) as T; +} + +function sanitizeFailedTask(task: PersistedForgetTask, file: string) { + return { + file, + key: task.key, + namespace: task.namespace, + sessionId: task.sessionId, + memoryIdsCount: Array.isArray(task.memoryIds) ? task.memoryIds.length : 0, + createdAt: task.createdAt, + attempt: task.attempt ?? 0, + nextRunAt: task.nextRunAt, + lastError: task.lastError, + }; +} + +export class DurableForgetQueue { + private readonly log: Logger; + private readonly qdrant: QdrantStore; + private readonly neo4j: Neo4jStore; + private readonly updateQueue: DurableUpdateQueue; + private readonly concurrency: number; + private readonly baseDir: string; + private readonly pendingDir: string; + private readonly inflightDir: string; + private readonly doneDir: string; + private readonly failedDir: string; + private readonly maxAttempts: number; + private readonly retryBaseMs: number; + private readonly retryMaxMs: number; + private readonly keepDone: boolean; + private readonly retentionDays: number; + + private active = 0; + private stopped = false; + private readonly inflightKeys = new Set(); + private readonly pendingFilesByKey = new Map(); + private idleWaiters: Array<() => void> = []; + + constructor(params: { + log: Logger; + qdrant: QdrantStore; + neo4j: Neo4jStore; + updateQueue: DurableUpdateQueue; + concurrency: number; + dir: string; + maxAttempts: number; + retryBaseMs: number; + retryMaxMs: number; + keepDone: boolean; + retentionDays: number; + }) { + this.log = params.log; + this.qdrant = params.qdrant; + this.neo4j = params.neo4j; + this.updateQueue = params.updateQueue; + this.concurrency = Math.max(1, params.concurrency); + this.baseDir = params.dir; + this.pendingDir = path.join(this.baseDir, "pending"); + this.inflightDir = path.join(this.baseDir, "inflight"); + this.doneDir = path.join(this.baseDir, "done"); + this.failedDir = path.join(this.baseDir, "failed"); + this.maxAttempts = Math.max(1, params.maxAttempts); + this.retryBaseMs = Math.max(1, params.retryBaseMs); + this.retryMaxMs = Math.max(this.retryBaseMs, params.retryMaxMs); + this.keepDone = params.keepDone; + this.retentionDays = Math.max(1, params.retentionDays); + } + + async init(): Promise { + await Promise.all([ + ensureDir(this.pendingDir), + ensureDir(this.inflightDir), + ensureDir(this.doneDir), + ensureDir(this.failedDir), + ]); + + // Crash recovery. + const inflight = await fs.readdir(this.inflightDir).catch(() => []); + for (const name of inflight) { + const from = path.join(this.inflightDir, name); + const to = path.join(this.pendingDir, name); + try { + const task = await readJson(from); + const attempt = Math.max(1, (task.attempt ?? 0) + 1); + task.attempt = attempt; + task.nextRunAt = + Date.now() + backoffMs({ baseMs: this.retryBaseMs, maxMs: this.retryMaxMs, attempt }); + await atomicWriteJson(to, task); + await fs.rm(from, { force: true }); + } catch (err) { + this.log.warn({ err: String(err), file: from }, "failed to recover inflight forget task"); + } + } + + const pending = await fs.readdir(this.pendingDir).catch(() => []); + for (const name of pending) { + const file = path.join(this.pendingDir, name); + try { + const task = await readJson(file); + if (task?.kind !== "forget" || !task.key) { + continue; + } + const prev = this.pendingFilesByKey.get(task.key); + if (!prev) { + this.pendingFilesByKey.set(task.key, file); + } else { + const prevTask = await readJson(prev); + const prevScore = Number(prevTask.nextRunAt ?? 0); + const nextScore = Number(task.nextRunAt ?? 0); + if (nextScore >= prevScore) { + this.pendingFilesByKey.set(task.key, file); + } + } + } catch { + // ignore + } + } + + void this.cleanupLoop(); + void this.pump(); + } + + stop(): void { + this.stopped = true; + } + + stats(): { pendingApprox: number; active: number; inflightKeys: number } { + return { + pendingApprox: this.pendingFilesByKey.size, + active: this.active, + inflightKeys: this.inflightKeys.size, + }; + } + + private notifyIdle() { + if (this.active !== 0) { + return; + } + const waiters = this.idleWaiters; + this.idleWaiters = []; + waiters.forEach((w) => w()); + } + + async waitIdle(): Promise { + if (this.active === 0) { + return; + } + await new Promise((resolve) => this.idleWaiters.push(resolve)); + } + + async enqueue(req: ForgetRequest): Promise<{ status: "queued"; key: string; taskId: string }> { + const namespace = req.namespace?.trim() || "default"; + const sessionId = req.sessionId?.trim() || undefined; + const memoryIds = Array.isArray(req.memoryIds) + ? req.memoryIds.map((x) => x.trim()).filter(Boolean) + : undefined; + const key = sessionId + ? `${namespace}::${sessionId}` + : `${namespace}::ids::${stableHash(JSON.stringify(memoryIds ?? []))}`; + + const now = new Date(); + const task: PersistedForgetTask = { + kind: "forget", + id: crypto.randomUUID(), + key, + namespace, + sessionId, + memoryIds, + createdAt: now.toISOString(), + attempt: 0, + nextRunAt: Date.now(), + }; + + const fileName = `${now.toISOString().replace(/[:.]/g, "-")}-${task.id}.json`; + const filePath = path.join(this.pendingDir, fileName); + await atomicWriteJson(filePath, task); + this.pendingFilesByKey.set(key, filePath); + return { status: "queued", key, taskId: task.id }; + } + + async listFailed(params: { limit: number }) { + const names = await fs.readdir(this.failedDir).catch(() => []); + const sorted = names.toSorted((a, b) => a.localeCompare(b)); + const out: Array> = []; + for (const name of sorted) { + if (out.length >= Math.max(1, params.limit)) { + break; + } + const filePath = path.join(this.failedDir, name); + try { + const task = await readJson(filePath); + if (!task || task.kind !== "forget") { + continue; + } + out.push(sanitizeFailedTask(task, name)); + } catch { + // ignore + } + } + return out; + } + + async exportFailed(params: { file?: string; key?: string; limit: number }) { + if (params.file) { + const filePath = path.join(this.failedDir, params.file); + const task = await readJson(filePath); + return { mode: "file" as const, item: sanitizeFailedTask(task, params.file) }; + } + if (params.key) { + const items = await this.listFailed({ limit: params.limit }); + return { + mode: "key" as const, + key: params.key, + items: items.filter((i) => i.key === params.key), + }; + } + return { mode: "empty" as const }; + } + + async retryFailed(params: { + file: string; + }): Promise<{ status: "requeued" } | { status: "not_found" }> { + const from = path.join(this.failedDir, params.file); + const exists = await fs + .stat(from) + .then(() => true) + .catch(() => false); + if (!exists) { + return { status: "not_found" }; + } + const task = await readJson(from); + task.lastError = undefined; + task.nextRunAt = Date.now(); + const to = path.join(this.pendingDir, params.file); + await atomicWriteJson(to, task); + await fs.rm(from, { force: true }); + this.pendingFilesByKey.set(task.key, to); + return { status: "requeued" }; + } + + async retryFailedByKey(params: { key: string; limit: number; dryRun: boolean }) { + const names = await fs.readdir(this.failedDir).catch(() => []); + let matched = 0; + let retried = 0; + for (const name of names.toSorted((a, b) => a.localeCompare(b))) { + if (retried >= Math.max(1, params.limit)) { + break; + } + const filePath = path.join(this.failedDir, name); + try { + const task = await readJson(filePath); + if (!task || task.kind !== "forget") { + continue; + } + if (task.key !== params.key) { + continue; + } + matched += 1; + if (params.dryRun) { + continue; + } + const out = await this.retryFailed({ file: name }); + if (out.status === "requeued") { + retried += 1; + } + } catch { + // ignore + } + } + return { status: "ok" as const, matched, retried }; + } + + private async cleanupLoop() { + const retentionMs = this.retentionDays * 24 * 3600_000; + while (!this.stopped) { + await sleep(60_000); + if (!this.keepDone) { + continue; + } + const now = Date.now(); + const names = await fs.readdir(this.doneDir).catch(() => []); + for (const name of names) { + const filePath = path.join(this.doneDir, name); + try { + const st = await fs.stat(filePath); + if (now - st.mtimeMs > retentionMs) { + await fs.rm(filePath, { force: true }); + } + } catch { + // ignore + } + } + } + } + + private async pump() { + while (!this.stopped) { + if (this.active >= this.concurrency) { + await sleep(50); + continue; + } + const due = await this.pickDueTask(); + if (!due) { + await sleep(100); + continue; + } + void this.runTask(due).finally(() => { + this.active = Math.max(0, this.active - 1); + this.notifyIdle(); + }); + } + } + + private async pickDueTask(): Promise<{ file: string; task: PersistedForgetTask } | null> { + const entries = Array.from(this.pendingFilesByKey.entries()); + const now = Date.now(); + for (const [key, file] of entries) { + if (this.inflightKeys.has(key)) { + continue; + } + try { + const task = await readJson(file); + if (!task || task.kind !== "forget") { + continue; + } + if ((task.nextRunAt ?? 0) > now) { + continue; + } + return { file, task }; + } catch { + // ignore + } + } + return null; + } + + private async runTask(picked: { file: string; task: PersistedForgetTask }) { + this.active += 1; + this.inflightKeys.add(picked.task.key); + const fileName = path.basename(picked.file); + const inflightPath = path.join(this.inflightDir, fileName); + try { + await fs.rename(picked.file, inflightPath); + } catch { + this.inflightKeys.delete(picked.task.key); + return; + } + + const task = await readJson(inflightPath).catch(() => picked.task); + try { + task.result = await this.processForget(task); + const donePath = path.join(this.doneDir, fileName); + await atomicWriteJson(donePath, task); + await fs.rm(inflightPath, { force: true }); + this.pendingFilesByKey.delete(task.key); + } catch (err) { + const attempt = Math.max(1, (task.attempt ?? 0) + 1); + task.attempt = attempt; + task.lastError = err instanceof Error ? err.message : String(err); + if (attempt >= this.maxAttempts) { + const failedPath = path.join(this.failedDir, fileName); + await atomicWriteJson(failedPath, task); + await fs.rm(inflightPath, { force: true }); + this.pendingFilesByKey.delete(task.key); + } else { + task.nextRunAt = + Date.now() + backoffMs({ baseMs: this.retryBaseMs, maxMs: this.retryMaxMs, attempt }); + const pendingPath = path.join(this.pendingDir, fileName); + await atomicWriteJson(pendingPath, task); + await fs.rm(inflightPath, { force: true }); + this.pendingFilesByKey.set(task.key, pendingPath); + } + } finally { + this.inflightKeys.delete(task.key); + } + } + + private async processForget( + task: PersistedForgetTask, + ): Promise> { + const out: NonNullable = { + qdrant: {}, + neo4j: {}, + queue: { ok: true, cancelled: 0 }, + deletedNeo4j: 0, + }; + + if (task.sessionId) { + try { + await this.qdrant.deleteBySession({ namespace: task.namespace, sessionId: task.sessionId }); + out.qdrant!.bySession = { ok: true }; + } catch (err) { + out.qdrant!.bySession = { + ok: false, + error: err instanceof Error ? err.message : String(err), + }; + } + try { + const d = await this.neo4j.deleteMemoriesBySession({ + namespace: task.namespace, + sessionId: task.sessionId, + }); + out.deletedNeo4j = (out.deletedNeo4j ?? 0) + d; + out.neo4j!.bySession = { ok: true, deleted: d }; + } catch (err) { + out.neo4j!.bySession = { + ok: false, + error: err instanceof Error ? err.message : String(err), + }; + } + try { + const cancelled = await this.updateQueue.cancelBySession({ + namespace: task.namespace, + sessionId: task.sessionId, + }); + out.queue = { ok: true, cancelled }; + } catch (err) { + out.queue = { + ok: false, + cancelled: 0, + error: err instanceof Error ? err.message : String(err), + }; + } + } + + const ids = Array.isArray(task.memoryIds) ? task.memoryIds : []; + if (ids.length > 0) { + try { + const d = await this.qdrant.deleteByIds({ ids }); + out.qdrant!.byIds = { ok: true, deleted: d }; + } catch (err) { + out.qdrant!.byIds = { ok: false, error: err instanceof Error ? err.message : String(err) }; + } + try { + const d = await this.neo4j.deleteMemoriesByIds({ namespace: task.namespace, ids }); + out.deletedNeo4j = (out.deletedNeo4j ?? 0) + d; + out.neo4j!.byIds = { ok: true, deleted: d }; + } catch (err) { + out.neo4j!.byIds = { ok: false, error: err instanceof Error ? err.message : String(err) }; + } + } + + return out; + } +} diff --git a/packages/deep-memory-server/src/durable-update-queue.test.ts b/packages/deep-memory-server/src/durable-update-queue.test.ts new file mode 100644 index 0000000000000..da486bd2f0e8e --- /dev/null +++ b/packages/deep-memory-server/src/durable-update-queue.test.ts @@ -0,0 +1,236 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import pino from "pino"; +import { describe, expect, it } from "vitest"; +import { DurableUpdateQueue } from "./durable-update-queue.js"; + +describe("DurableUpdateQueue", () => { + it("persists tasks and processes them", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "deepmem-queue-")); + const log = pino({ level: "silent" }); + + let processed = 0; + const updater = { + update: async () => { + processed += 1; + return { status: "processed", memories_added: 1, memories_filtered: 0 }; + }, + }; + + const queue = new DurableUpdateQueue({ + log, + updater: updater as unknown as never, + concurrency: 1, + namespaceConcurrency: 0, + dir, + maxAttempts: 3, + retryBaseMs: 10, + retryMaxMs: 50, + keepDone: true, + retentionDays: 1, + maxTaskBytes: 1024 * 1024, + }); + await queue.init(); + + await queue.enqueue({ + namespace: "default", + sessionId: "s1", + messages: [{ role: "user", content: "hi" }], + }); + const ok = await queue.onIdle({ timeoutMs: 5_000 }); + expect(ok).toBe(true); + expect(processed).toBe(1); + + const doneDir = path.join(dir, "done"); + const done = await fs.readdir(doneDir); + expect(done.length).toBeGreaterThanOrEqual(1); + }); + + it("recovers inflight tasks on restart", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "deepmem-queue-")); + const log = pino({ level: "silent" }); + + // First queue: enqueue and simulate an inflight crash by moving file ourselves. + const updater = { + update: async () => ({ status: "processed", memories_added: 1, memories_filtered: 0 }), + }; + const q1 = new DurableUpdateQueue({ + log, + updater: updater as unknown as never, + concurrency: 1, + namespaceConcurrency: 0, + dir, + maxAttempts: 3, + retryBaseMs: 10, + retryMaxMs: 50, + keepDone: false, + retentionDays: 1, + maxTaskBytes: 1024 * 1024, + }); + await q1.init(); + await q1.enqueue({ + namespace: "default", + sessionId: "s1", + messages: [{ role: "user", content: "hi" }], + }); + await q1.onIdle({ timeoutMs: 5_000 }); + + // Now create an inflight file to be recovered. + const pendingDir = path.join(dir, "pending"); + const inflightDir = path.join(dir, "inflight"); + const fakeFile = path.join(inflightDir, "fake.json"); + await fs.writeFile( + fakeFile, + JSON.stringify({ + kind: "update", + id: "x", + key: "default::s2", + namespace: "default", + sessionId: "s2", + transcriptHash: "h", + messageCount: 1, + createdAt: new Date().toISOString(), + attempt: 0, + nextRunAt: Date.now(), + messages: [{ role: "user", content: "yo" }], + }), + "utf8", + ); + + // Restart: should move inflight back to pending. + const q2 = new DurableUpdateQueue({ + log, + updater: updater as unknown as never, + concurrency: 1, + dir, + maxAttempts: 3, + retryBaseMs: 10, + retryMaxMs: 50, + keepDone: false, + retentionDays: 1, + maxTaskBytes: 1024 * 1024, + }); + await q2.init(); + const pending = await fs.readdir(pendingDir); + expect(pending.some((n) => n === "fake.json")).toBe(true); + }); + + it("exports failed tasks without messages", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "deepmem-queue-")); + const log = pino({ level: "silent" }); + const updater = { + update: async () => ({ status: "processed", memories_added: 1, memories_filtered: 0 }), + }; + const q = new DurableUpdateQueue({ + log, + updater: updater as unknown as never, + concurrency: 1, + dir, + maxAttempts: 1, + retryBaseMs: 10, + retryMaxMs: 50, + keepDone: false, + retentionDays: 1, + maxTaskBytes: 1024 * 1024, + }); + await q.init(); + + const failedDir = path.join(dir, "failed"); + await fs.writeFile( + path.join(failedDir, "x.json"), + JSON.stringify({ + kind: "update", + id: "x", + key: "default::s1", + namespace: "default", + sessionId: "s1", + transcriptHash: "h", + messageCount: 1, + createdAt: new Date().toISOString(), + attempt: 10, + nextRunAt: Date.now(), + lastError: "boom", + messages_gzip_base64: "H4sIAAAAAAAAAwMAAAAAAAAAAAA=", // gzipped empty payload placeholder + }), + "utf8", + ); + + const out = await q.exportFailed({ limit: 10 }); + expect(out.mode).toBe("list"); + if (out.mode === "list") { + const first = out.items[0]; + expect(first?.file).toBe("x.json"); + expect((first as Record | undefined)?.messages).toBeUndefined(); + expect(first?.lastError).toBe("boom"); + } + }); + + it("retries failed tasks by key with limit", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "deepmem-queue-")); + const log = pino({ level: "silent" }); + const updater = { + update: async () => ({ status: "processed", memories_added: 1, memories_filtered: 0 }), + }; + const q = new DurableUpdateQueue({ + log, + updater: updater as unknown as never, + concurrency: 1, + dir, + maxAttempts: 1, + retryBaseMs: 10, + retryMaxMs: 50, + keepDone: false, + retentionDays: 1, + maxTaskBytes: 1024 * 1024, + }); + await q.init(); + + const failedDir = path.join(dir, "failed"); + const pendingDir = path.join(dir, "pending"); + const now = Date.now(); + await fs.writeFile( + path.join(failedDir, "a.json"), + JSON.stringify({ + kind: "update", + id: "a", + key: "default::s1", + namespace: "default", + sessionId: "s1", + transcriptHash: "h1", + messageCount: 1, + createdAt: new Date().toISOString(), + attempt: 10, + nextRunAt: now, + lastError: "boom", + messages_gzip_base64: "H4sIAAAAAAAAAwMAAAAAAAAAAAA=", + }), + "utf8", + ); + await fs.writeFile( + path.join(failedDir, "b.json"), + JSON.stringify({ + kind: "update", + id: "b", + key: "default::s1", + namespace: "default", + sessionId: "s1", + transcriptHash: "h2", + messageCount: 1, + createdAt: new Date().toISOString(), + attempt: 10, + nextRunAt: now, + lastError: "boom", + messages_gzip_base64: "H4sIAAAAAAAAAwMAAAAAAAAAAAA=", + }), + "utf8", + ); + + const out = await q.retryFailedByKey({ key: "default::s1", limit: 1, dryRun: false }); + expect(out.matched).toBeGreaterThanOrEqual(1); + expect(out.retried).toBe(1); + + const pending = await fs.readdir(pendingDir); + expect(pending.length).toBe(1); + }); +}); diff --git a/packages/deep-memory-server/src/durable-update-queue.ts b/packages/deep-memory-server/src/durable-update-queue.ts new file mode 100644 index 0000000000000..e9f499e9d9b47 --- /dev/null +++ b/packages/deep-memory-server/src/durable-update-queue.ts @@ -0,0 +1,673 @@ +import crypto from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { gzipSync, gunzipSync } from "node:zlib"; +import type { Logger } from "pino"; +import type { UpdateMemoryIndexResponse } from "./types.js"; +import type { DeepMemoryUpdater } from "./updater.js"; + +type UpdateRequest = { + namespace: string; + sessionId: string; + messages: unknown[]; + notBeforeMs?: number; + returnMemoryIds?: { + max: number; + }; +}; + +type PersistedUpdateTask = { + kind: "update"; + id: string; // unique + key: string; // namespace::sessionId + namespace: string; + sessionId: string; + transcriptHash: string; + messageCount: number; + createdAt: string; // ISO + attempt: number; + nextRunAt: number; // epoch ms + lastError?: string; + // New format: compressed to reduce disk usage. + messages_gzip_base64?: string; + // Legacy format (back-compat). + messages?: unknown[]; +}; + +function sleep(ms: number) { + return new Promise((r) => setTimeout(r, ms)); +} + +function stableTranscriptHash(messages: unknown[]): { hash: string; count: number } { + const count = Array.isArray(messages) ? messages.length : 0; + const hash = crypto + .createHash("sha256") + .update(JSON.stringify(messages ?? [])) + .digest("hex"); + return { hash, count }; +} + +function encodeMessages(messages: unknown[]): { b64: string; bytes: number } { + const json = JSON.stringify(messages ?? []); + const gz = gzipSync(Buffer.from(json, "utf8"), { level: 9 }); + return { b64: gz.toString("base64"), bytes: gz.length }; +} + +function decodeMessages(task: PersistedUpdateTask): unknown[] { + if (task.messages_gzip_base64) { + const buf = Buffer.from(task.messages_gzip_base64, "base64"); + const raw = gunzipSync(buf).toString("utf8"); + const parsed = JSON.parse(raw); + return Array.isArray(parsed) ? parsed : []; + } + return Array.isArray(task.messages) ? task.messages : []; +} + +function backoffMs(params: { baseMs: number; maxMs: number; attempt: number }): number { + // attempt starts at 1. Exponential backoff with jitter. + const exp = Math.min(20, Math.max(0, params.attempt - 1)); + const raw = Math.min(params.maxMs, params.baseMs * 2 ** exp); + const jitter = Math.floor(Math.random() * Math.min(250, Math.max(10, raw / 10))); + return Math.min(params.maxMs, raw + jitter); +} + +async function ensureDir(dir: string) { + await fs.mkdir(dir, { recursive: true }); +} + +async function atomicWriteJson(filePath: string, value: unknown) { + const dir = path.dirname(filePath); + await ensureDir(dir); + const tmp = `${filePath}.tmp-${crypto.randomUUID()}`; + const data = JSON.stringify(value); + const handle = await fs.open(tmp, "w"); + try { + await handle.writeFile(data, "utf8"); + await handle.sync(); + } finally { + await handle.close(); + } + await fs.rename(tmp, filePath); +} + +async function readJson(filePath: string): Promise { + const raw = await fs.readFile(filePath, "utf8"); + return JSON.parse(raw) as T; +} + +function sanitizeFailedTask(task: PersistedUpdateTask, file: string) { + return { + file, + key: task.key, + namespace: task.namespace, + sessionId: task.sessionId, + transcriptHash: task.transcriptHash, + messageCount: task.messageCount, + createdAt: task.createdAt, + attempt: task.attempt ?? 0, + nextRunAt: task.nextRunAt, + lastError: task.lastError, + }; +} + +export class DurableUpdateQueue { + private readonly log: Logger; + private readonly updater: DeepMemoryUpdater; + private readonly concurrency: number; + private readonly namespaceConcurrency: number; + private readonly baseDir: string; + private readonly pendingDir: string; + private readonly inflightDir: string; + private readonly doneDir: string; + private readonly failedDir: string; + private readonly maxAttempts: number; + private readonly retryBaseMs: number; + private readonly retryMaxMs: number; + private readonly keepDone: boolean; + private readonly retentionDays: number; + private readonly maxTaskBytes: number; + + private active = 0; + private stopped = false; + private readonly inflightKeys = new Set(); + private readonly inflightNamespaces = new Map(); + private readonly pendingFilesByKey = new Map(); // best-effort "latest" + + private idleWaiters: Array<() => void> = []; + + constructor(params: { + log: Logger; + updater: DeepMemoryUpdater; + concurrency: number; + namespaceConcurrency?: number; + dir: string; + maxAttempts: number; + retryBaseMs: number; + retryMaxMs: number; + keepDone: boolean; + retentionDays: number; + maxTaskBytes: number; + }) { + this.log = params.log; + this.updater = params.updater; + this.concurrency = Math.max(1, params.concurrency); + this.namespaceConcurrency = Math.max(0, params.namespaceConcurrency ?? 0); + this.baseDir = params.dir; + this.pendingDir = path.join(this.baseDir, "pending"); + this.inflightDir = path.join(this.baseDir, "inflight"); + this.doneDir = path.join(this.baseDir, "done"); + this.failedDir = path.join(this.baseDir, "failed"); + this.maxAttempts = Math.max(1, params.maxAttempts); + this.retryBaseMs = Math.max(1, params.retryBaseMs); + this.retryMaxMs = Math.max(this.retryBaseMs, params.retryMaxMs); + this.keepDone = params.keepDone; + this.retentionDays = Math.max(1, params.retentionDays); + this.maxTaskBytes = Math.max(1, params.maxTaskBytes); + } + + async init(): Promise { + await Promise.all([ + ensureDir(this.pendingDir), + ensureDir(this.inflightDir), + ensureDir(this.doneDir), + ensureDir(this.failedDir), + ]); + + // Crash recovery: anything in inflight goes back to pending with a backoff. + const inflight = await fs.readdir(this.inflightDir).catch(() => []); + for (const name of inflight) { + const from = path.join(this.inflightDir, name); + const to = path.join(this.pendingDir, name); + try { + const task = await readJson(from); + const attempt = Math.max(1, (task.attempt ?? 0) + 1); + task.attempt = attempt; + task.nextRunAt = + Date.now() + backoffMs({ baseMs: this.retryBaseMs, maxMs: this.retryMaxMs, attempt }); + await atomicWriteJson(to, task); + await fs.rm(from, { force: true }); + } catch (err) { + this.log.warn({ err: String(err), file: from }, "failed to recover inflight task"); + // Leave it for manual inspection. + } + } + + // Build best-effort index so we can replace older per-key tasks. + const pending = await fs.readdir(this.pendingDir).catch(() => []); + for (const name of pending) { + const file = path.join(this.pendingDir, name); + try { + const task = await readJson(file); + if (task?.kind !== "update" || !task.key) { + continue; + } + const prev = this.pendingFilesByKey.get(task.key); + if (!prev) { + this.pendingFilesByKey.set(task.key, file); + } else { + // Keep the newest by nextRunAt/createdAt (best-effort). + const prevTask = await readJson(prev); + const prevScore = Number(prevTask.nextRunAt ?? 0); + const nextScore = Number(task.nextRunAt ?? 0); + if (nextScore >= prevScore) { + this.pendingFilesByKey.set(task.key, file); + } + } + } catch { + // ignore + } + } + + // Periodic cleanup of done tasks. + void this.cleanupLoop(); + + // Start workers. + void this.pump(); + } + + stop(): void { + this.stopped = true; + } + + stats(): { pendingApprox: number; active: number; inflightKeys: number } { + return { + pendingApprox: this.pendingFilesByKey.size, + active: this.active, + inflightKeys: this.inflightKeys.size, + }; + } + + async listFailed(params: { limit: number }): Promise< + Array<{ + file: string; + key: string; + attempt: number; + lastError?: string; + createdAt: string; + nextRunAt: number; + }> + > { + const names = await fs.readdir(this.failedDir).catch(() => []); + const out: Array<{ + file: string; + key: string; + attempt: number; + lastError?: string; + createdAt: string; + nextRunAt: number; + }> = []; + for (const name of names.slice(0, Math.max(1, params.limit))) { + const filePath = path.join(this.failedDir, name); + try { + const task = await readJson(filePath); + if (!task || task.kind !== "update") { + continue; + } + out.push({ + file: name, + key: task.key, + attempt: task.attempt ?? 0, + lastError: task.lastError, + createdAt: task.createdAt, + nextRunAt: task.nextRunAt, + }); + } catch { + // ignore + } + } + return out; + } + + async exportFailed(params: { + file?: string; + key?: string; + limit: number; + }): Promise< + | { mode: "file"; item: ReturnType } + | { mode: "list"; items: Array> } + | { mode: "empty" } + > { + const file = params.file?.trim(); + if (file) { + const name = path.basename(file); + const filePath = path.join(this.failedDir, name); + try { + const task = await readJson(filePath); + if (!task || task.kind !== "update") { + return { mode: "empty" }; + } + return { mode: "file", item: sanitizeFailedTask(task, name) }; + } catch { + return { mode: "empty" }; + } + } + const key = params.key?.trim(); + const names = await fs.readdir(this.failedDir).catch(() => []); + const items: Array> = []; + for (const name of names) { + const filePath = path.join(this.failedDir, name); + try { + const task = await readJson(filePath); + if (!task || task.kind !== "update") { + continue; + } + if (key && task.key !== key) { + continue; + } + items.push(sanitizeFailedTask(task, name)); + if (items.length >= Math.max(1, params.limit)) { + break; + } + } catch { + // ignore + } + } + return { mode: "list", items }; + } + + async retryFailed(params: { file: string }): Promise<{ status: "requeued" | "not_found" }> { + const name = path.basename(params.file); + const from = path.join(this.failedDir, name); + try { + const task = await readJson(from); + if (!task || task.kind !== "update") { + return { status: "not_found" }; + } + // Reset schedule; keep attempt counter for visibility. + task.nextRunAt = Date.now(); + task.lastError = undefined; + const to = path.join(this.pendingDir, name); + await atomicWriteJson(to, task); + await fs.rm(from, { force: true }); + + const prev = this.pendingFilesByKey.get(task.key); + this.pendingFilesByKey.set(task.key, to); + if (prev && prev !== to) { + void fs.rm(prev, { force: true }).catch(() => {}); + } + void this.pump(); + return { status: "requeued" }; + } catch { + return { status: "not_found" }; + } + } + + async retryFailedByKey(params: { + key: string; + limit: number; + dryRun: boolean; + }): Promise<{ status: "ok"; matched: number; retried: number }> { + const key = params.key.trim(); + if (!key) { + return { status: "ok", matched: 0, retried: 0 }; + } + const names = await fs.readdir(this.failedDir).catch(() => []); + let matched = 0; + let retried = 0; + for (const name of names) { + const filePath = path.join(this.failedDir, name); + try { + const task = await readJson(filePath); + if (!task || task.kind !== "update") { + continue; + } + if (task.key !== key) { + continue; + } + matched += 1; + if (params.dryRun) { + continue; + } + const out = await this.retryFailed({ file: name }); + if (out.status === "requeued") { + retried += 1; + } + if (retried >= Math.max(1, params.limit)) { + break; + } + } catch { + // ignore + } + } + return { status: "ok", matched, retried }; + } + + async cancelBySession(params: { namespace: string; sessionId: string }): Promise { + const key = `${params.namespace}::${params.sessionId}`; + const file = this.pendingFilesByKey.get(key); + if (!file) { + return 0; + } + this.pendingFilesByKey.delete(key); + try { + await fs.rm(file, { force: true }); + return 1; + } catch { + return 0; + } + } + + async enqueue( + req: UpdateRequest, + ): Promise<{ status: "queued"; key: string; transcriptHash: string }> { + const namespace = req.namespace?.trim() || "default"; + const key = `${namespace}::${req.sessionId}`; + const { hash, count } = stableTranscriptHash(req.messages); + + const existingPath = this.pendingFilesByKey.get(key); + if (existingPath) { + try { + const existing = await readJson(existingPath); + if (existing?.transcriptHash === hash) { + return { status: "queued", key, transcriptHash: hash }; + } + } catch { + // ignore + } + } + + const now = new Date(); + const encoded = encodeMessages(Array.isArray(req.messages) ? req.messages : []); + if (encoded.bytes > this.maxTaskBytes) { + throw new Error( + `queue task too large (${encoded.bytes} bytes gzipped > ${this.maxTaskBytes})`, + ); + } + const task: PersistedUpdateTask = { + kind: "update", + id: crypto.randomUUID(), + key, + namespace, + sessionId: req.sessionId, + transcriptHash: hash, + messageCount: count, + createdAt: now.toISOString(), + attempt: 0, + nextRunAt: Math.max(Date.now(), Number(req.notBeforeMs ?? Date.now()) || Date.now()), + messages_gzip_base64: encoded.b64, + }; + + // File name includes key hash so it is filesystem safe and stable. + const keyHash = crypto.createHash("sha256").update(key).digest("hex").slice(0, 16); + const fileName = `${keyHash}-${Date.now()}-${task.id}.json`; + const filePath = path.join(this.pendingDir, fileName); + + await atomicWriteJson(filePath, task); + + // Best-effort: keep only latest pending task per key (debounce/coalesce). + const prev = this.pendingFilesByKey.get(key); + this.pendingFilesByKey.set(key, filePath); + if (prev && prev !== filePath) { + void fs.rm(prev, { force: true }).catch(() => {}); + } + + void this.pump(); + return { status: "queued", key, transcriptHash: hash }; + } + + async runNow(req: UpdateRequest): Promise { + // Synchronous path still needs per-key mutual exclusion. + const key = `${req.namespace}::${req.sessionId}`; + while (this.inflightKeys.has(key)) { + await sleep(50); + } + this.inflightKeys.add(key); + try { + return await this.updater.update(req); + } finally { + this.inflightKeys.delete(key); + this.signalIdleIfNeeded(); + void this.pump(); + } + } + + async onIdle(params?: { timeoutMs?: number }): Promise { + const timeoutMs = params?.timeoutMs ?? 10_000; + if (this.active === 0 && this.pendingFilesByKey.size === 0) { + return true; + } + let timer: NodeJS.Timeout | null = null; + return await new Promise((resolve) => { + const done = () => { + if (timer) { + clearTimeout(timer); + } + resolve(true); + }; + this.idleWaiters.push(done); + timer = setTimeout(() => { + this.idleWaiters = this.idleWaiters.filter((w) => w !== done); + resolve(false); + }, timeoutMs); + }); + } + + private signalIdleIfNeeded() { + if (this.active !== 0 || this.pendingFilesByKey.size !== 0) { + return; + } + const waiters = this.idleWaiters; + this.idleWaiters = []; + for (const w of waiters) { + w(); + } + } + + private async cleanupLoop() { + while (!this.stopped) { + await sleep(30_000); + if (!this.keepDone) { + continue; + } + const cutoff = Date.now() - this.retentionDays * 24 * 3600_000; + const entries = await fs.readdir(this.doneDir).catch(() => []); + for (const name of entries) { + const file = path.join(this.doneDir, name); + try { + const stat = await fs.stat(file); + if (stat.mtimeMs < cutoff) { + await fs.rm(file, { force: true }); + } + } catch { + // ignore + } + } + } + } + + private async pump() { + if (this.stopped) { + return; + } + while (this.active < this.concurrency) { + const next = await this.pickRunnableTask(); + if (!next) { + this.signalIdleIfNeeded(); + return; + } + this.active += 1; + void this.runTask(next) + .catch((err) => { + this.log.warn( + { err: String(err), file: next.filePath }, + "task execution failed (unexpected)", + ); + }) + .finally(() => { + this.active -= 1; + this.signalIdleIfNeeded(); + void this.pump(); + }); + } + } + + private async pickRunnableTask(): Promise<{ + filePath: string; + task: PersistedUpdateTask; + } | null> { + // Best-effort: pick first runnable among latest-by-key. + const now = Date.now(); + for (const [key, filePath] of this.pendingFilesByKey.entries()) { + if (this.inflightKeys.has(key)) { + continue; + } + try { + const task = await readJson(filePath); + if (!task || task.kind !== "update") { + this.pendingFilesByKey.delete(key); + continue; + } + if ((task.nextRunAt ?? 0) > now) { + continue; + } + if (this.namespaceConcurrency > 0) { + const activeNs = this.inflightNamespaces.get(task.namespace) ?? 0; + if (activeNs >= this.namespaceConcurrency) { + continue; + } + } + // Move to inflight atomically by rename to avoid duplicates across crashes. + const inflightPath = path.join(this.inflightDir, path.basename(filePath)); + await fs.rename(filePath, inflightPath); + this.pendingFilesByKey.delete(key); + task.attempt = Math.max(0, task.attempt ?? 0); + return { filePath: inflightPath, task }; + } catch { + // If file disappeared or unreadable, drop it. + this.pendingFilesByKey.delete(key); + } + } + return null; + } + + private async runTask(params: { filePath: string; task: PersistedUpdateTask }): Promise { + const task = params.task; + const key = task.key; + const ns = task.namespace; + if (this.inflightKeys.has(key)) { + // Another runner took it (shouldn't happen). Put back. + await fs.rename(params.filePath, path.join(this.pendingDir, path.basename(params.filePath))); + this.pendingFilesByKey.set(key, path.join(this.pendingDir, path.basename(params.filePath))); + return; + } + this.inflightKeys.add(key); + if (this.namespaceConcurrency > 0) { + this.inflightNamespaces.set(ns, (this.inflightNamespaces.get(ns) ?? 0) + 1); + } + try { + const attempt = Math.max(1, (task.attempt ?? 0) + 1); + task.attempt = attempt; + const messages = decodeMessages(task); + const result = await this.updater.update({ + namespace: task.namespace, + sessionId: task.sessionId, + messages, + }); + if (result.status === "error") { + throw new Error(result.error ?? "update returned error"); + } + + if (this.keepDone) { + await fs.rename(params.filePath, path.join(this.doneDir, path.basename(params.filePath))); + } else { + await fs.rm(params.filePath, { force: true }); + } + } catch (err) { + const attempt = Math.max(1, task.attempt ?? 1); + const lastError = String(err); + if (attempt >= this.maxAttempts) { + const failed: PersistedUpdateTask = { + ...task, + lastError, + nextRunAt: Date.now(), + }; + await atomicWriteJson(path.join(this.failedDir, path.basename(params.filePath)), failed); + await fs.rm(params.filePath, { force: true }); + this.log.error({ key, attempt, lastError }, "task moved to failed"); + return; + } + + const delay = backoffMs({ baseMs: this.retryBaseMs, maxMs: this.retryMaxMs, attempt }); + const retryTask: PersistedUpdateTask = { + ...task, + lastError, + nextRunAt: Date.now() + delay, + }; + const pendingPath = path.join(this.pendingDir, path.basename(params.filePath)); + await atomicWriteJson(pendingPath, retryTask); + await fs.rm(params.filePath, { force: true }); + this.pendingFilesByKey.set(key, pendingPath); + this.log.warn({ key, attempt, delay, lastError }, "task retry scheduled"); + } finally { + this.inflightKeys.delete(key); + if (this.namespaceConcurrency > 0) { + const next = (this.inflightNamespaces.get(ns) ?? 1) - 1; + if (next <= 0) { + this.inflightNamespaces.delete(ns); + } else { + this.inflightNamespaces.set(ns, next); + } + } + } + } +} diff --git a/packages/deep-memory-server/src/embeddings.ts b/packages/deep-memory-server/src/embeddings.ts new file mode 100644 index 0000000000000..d8fc0630936e8 --- /dev/null +++ b/packages/deep-memory-server/src/embeddings.ts @@ -0,0 +1,55 @@ +import { pipeline } from "@xenova/transformers"; + +type FeatureExtractor = ( + text: string, + params: { pooling: "mean"; normalize: boolean }, +) => Promise; + +let pipe: FeatureExtractor | null = null; + +export class EmbeddingModel { + private readonly modelId: string; + private readonly dims: number; + + constructor(params: { modelId: string; dims: number }) { + this.modelId = params.modelId; + this.dims = params.dims; + } + + async ensureLoaded(): Promise { + if (pipe) { + return; + } + pipe = await pipeline("feature-extraction", this.modelId, { + quantized: true, + }); + } + + /** + * Returns a normalized embedding vector (cosine-ready). + * We use mean pooling over token embeddings. + */ + async embed(text: string): Promise { + const cleaned = text.trim(); + if (!cleaned) { + return Array.from({ length: this.dims }, () => 0); + } + await this.ensureLoaded(); + const result = await pipe!(cleaned, { pooling: "mean", normalize: true }); + // transformers.js may return a nested typed array; normalize to number[] + const data = + result && typeof result === "object" && "data" in result + ? (result as Record).data + : undefined; + const arr = Array.isArray(data) ? data : data instanceof Float32Array ? Array.from(data) : []; + if (arr.length !== this.dims) { + // Best-effort: pad/truncate to expected dims. + const out = arr.slice(0, this.dims); + while (out.length < this.dims) { + out.push(0); + } + return out; + } + return arr; + } +} diff --git a/packages/deep-memory-server/src/eval.ts b/packages/deep-memory-server/src/eval.ts new file mode 100644 index 0000000000000..fb666450af840 --- /dev/null +++ b/packages/deep-memory-server/src/eval.ts @@ -0,0 +1,342 @@ +import { readFile } from "node:fs/promises"; +import { writeFile } from "node:fs/promises"; +import path from "node:path"; +import process from "node:process"; +import { z } from "zod"; +import { runEval } from "./eval/harness.js"; +import { createRetrieverForEval } from "./eval/stubs.js"; +import type { EvalDataset } from "./eval/types.js"; + +const DatasetSchema = z.object({ + version: z.number().int().positive(), + cases: z.array( + z.object({ + name: z.string(), + namespace: z.string(), + now: z.string(), + query: z.object({ + userInput: z.string(), + sessionId: z.string(), + entities: z.array(z.string()), + topics: z.array(z.string()), + maxMemories: z.number().int().positive(), + }), + qdrantHits: z.array( + z.object({ + id: z.string(), + score: z.number(), + payload: z.object({ + content: z.string(), + importance: z.number().optional(), + frequency: z.number().optional(), + created_at: z.string(), + updated_at: z.string().optional(), + kind: z.string().optional(), + memory_key: z.string().optional(), + subject: z.string().optional(), + expires_at: z.string().optional(), + confidence: z.number().optional(), + }), + }), + ), + neo4jHits: z.array( + z.object({ + id: z.string(), + content: z.string(), + importance: z.number(), + frequency: z.number(), + lastSeenAt: z.string(), + relationScore: z.number(), + kind: z.string().optional(), + memoryKey: z.string().optional(), + subject: z.string().optional(), + expiresAt: z.string().optional(), + confidence: z.number().optional(), + }), + ), + expect: z.object({ + includeIds: z.array(z.string()).optional(), + excludeIds: z.array(z.string()).optional(), + top1Id: z.string().optional(), + uniqueByMemoryKey: z.boolean().optional(), + }), + }), + ), +}); + +type Knobs = { + minSemanticScore: number; + semanticWeight: number; + relationWeight: number; + decayHalfLifeDays: number; + importanceBoost: number; + frequencyBoost: number; +}; + +type Args = { + datasetPath: string; + sweep: boolean; + outPath?: string; + comparePath?: string; +}; + +type EvalRunSnapshot = { + version: 1; + ts: string; + datasetPath: string; + knobs: Knobs; + summary: { ok: boolean; cases: number; avgRecallAtK: number; avgNdcgAtK: number }; + cases: Array<{ + name: string; + ok: boolean; + recallAtK: number; + ndcgAtK: number; + top1Ok: boolean; + errors: string[]; + expected: Record; + retrievedIds: string[]; + retrievedMemoryKeys: Array; + }>; +}; + +function parseArgs(argv: string[]): Args { + let datasetPath = path.join(process.cwd(), "eval", "dataset.sample.json"); + let sweep = false; + let outPath: string | undefined; + let comparePath: string | undefined; + for (let i = 0; i < argv.length; i += 1) { + const t = argv[i]?.trim(); + if (!t) { + continue; + } + if (t === "--dataset" || t === "-d") { + const v = argv[i + 1]?.trim(); + if (v) { + datasetPath = v; + } + i += 1; + continue; + } + if (t === "--sweep") { + sweep = true; + continue; + } + if (t === "--out") { + const v = argv[i + 1]?.trim(); + if (v) { + outPath = v; + } + i += 1; + continue; + } + if (t === "--compare") { + const v = argv[i + 1]?.trim(); + if (v) { + comparePath = v; + } + i += 1; + continue; + } + if (t === "--help" || t === "-h") { + // eslint-disable-next-line no-console + console.log(` +Usage: pnpm --dir packages/deep-memory-server eval -- [options] + +Options: + --dataset, -d Dataset JSON file (default: eval/dataset.sample.json) + --sweep Run a small parameter sweep (thresholds/weights) + --out Write a JSON snapshot of results (for baseline/compare) + --compare Compare current run vs a baseline snapshot JSON + --help, -h Show help +`); + process.exit(0); + } + } + return { datasetPath, sweep, outPath, comparePath }; +} + +async function loadDataset(datasetPath: string): Promise { + const raw = await readFile(datasetPath, "utf-8"); + const parsed = DatasetSchema.safeParse(JSON.parse(raw) as unknown); + if (!parsed.success) { + throw new Error(`Invalid dataset: ${parsed.error.message}`); + } + return parsed.data as EvalDataset; +} + +async function runOnce(dataset: EvalDataset, knobs: Knobs) { + const results = await Promise.all( + dataset.cases.map(async (c) => { + const retriever = createRetrieverForEval({ evalCase: c, knobs }); + return await runEval({ retriever, cases: [c] }); + }), + ); + const cases = results.flatMap((r) => r.cases); + const ok = cases.every((c) => c.ok); + const avgRecallAtK = + cases.length === 0 ? 1 : cases.reduce((s, r) => s + r.recallAtK, 0) / cases.length; + const avgNdcgAtK = + cases.length === 0 ? 1 : cases.reduce((s, r) => s + r.ndcgAtK, 0) / cases.length; + return { ok, cases, avgRecallAtK, avgNdcgAtK }; +} + +function formatScore(x: number): string { + return `${(x * 100).toFixed(1)}%`; +} + +function formatDelta(x: number): string { + const pct = (x * 100).toFixed(2); + return x >= 0 ? `+${pct}pp` : `${pct}pp`; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const dataset = await loadDataset(args.datasetPath); + + const base: Knobs = { + minSemanticScore: 0.6, + semanticWeight: 0.6, + relationWeight: 0.4, + decayHalfLifeDays: 90, + importanceBoost: 0.3, + frequencyBoost: 0.2, + }; + + if (!args.sweep) { + const out = await runOnce(dataset, base); + // eslint-disable-next-line no-console + console.log( + `eval: ok=${out.ok} cases=${out.cases.length} recall@k=${formatScore(out.avgRecallAtK)} ndcg@k=${formatScore(out.avgNdcgAtK)}`, + ); + const snapshot: EvalRunSnapshot = { + version: 1, + ts: new Date().toISOString(), + datasetPath: args.datasetPath, + knobs: base, + summary: { + ok: out.ok, + cases: out.cases.length, + avgRecallAtK: out.avgRecallAtK, + avgNdcgAtK: out.avgNdcgAtK, + }, + cases: out.cases.map((c) => ({ + name: c.name, + ok: c.ok, + recallAtK: c.recallAtK, + ndcgAtK: c.ndcgAtK, + top1Ok: c.top1Ok, + errors: c.errors, + expected: c.expected as unknown as Record, + retrievedIds: c.retrievedIds, + retrievedMemoryKeys: c.retrievedMemoryKeys, + })), + }; + if (args.outPath) { + await writeFile(args.outPath, `${JSON.stringify(snapshot, null, 2)}\n`, "utf8"); + // eslint-disable-next-line no-console + console.log(`eval: wrote snapshot to ${args.outPath}`); + } + if (args.comparePath) { + const raw = await readFile(args.comparePath, "utf8"); + const baseline = JSON.parse(raw) as EvalRunSnapshot; + const baseByName = new Map(baseline.cases.map((c) => [c.name, c])); + // eslint-disable-next-line no-console + console.log( + `eval: compare baseline=${args.comparePath} (baseline ok=${baseline.summary.ok} ndcg@k=${formatScore(baseline.summary.avgNdcgAtK)})`, + ); + for (const c of snapshot.cases) { + const b = baseByName.get(c.name); + if (!b) { + // eslint-disable-next-line no-console + console.log(`- NEW: ${c.name}`); + continue; + } + const dRecall = c.recallAtK - b.recallAtK; + const dNdcg = c.ndcgAtK - b.ndcgAtK; + if (dRecall === 0 && dNdcg === 0 && c.ok === b.ok) { + continue; + } + // eslint-disable-next-line no-console + console.log( + `- ${c.name}: ok ${b.ok}→${c.ok} recall ${formatScore(b.recallAtK)}→${formatScore(c.recallAtK)} (${formatDelta(dRecall)}) ndcg ${formatScore(b.ndcgAtK)}→${formatScore(c.ndcgAtK)} (${formatDelta(dNdcg)})`, + ); + } + } + for (const c of snapshot.cases) { + if (c.ok) { + continue; + } + // eslint-disable-next-line no-console + console.log(`- FAIL: ${c.name}`); + // eslint-disable-next-line no-console + console.log(` expected: ${JSON.stringify(c.expected)}`); + // eslint-disable-next-line no-console + console.log(` retrievedIds: ${JSON.stringify(c.retrievedIds)}`); + // eslint-disable-next-line no-console + console.log(` retrievedMemoryKeys: ${JSON.stringify(c.retrievedMemoryKeys)}`); + for (const e of c.errors) { + // eslint-disable-next-line no-console + console.log(` - ${e}`); + } + } + process.exit(out.ok ? 0 : 1); + } + + const sweeps: Knobs[] = []; + for (const minSemanticScore of [0.4, 0.5, 0.6, 0.7]) { + for (const semanticWeight of [0.5, 0.6, 0.7]) { + const relationWeight = 1 - semanticWeight; + for (const decayHalfLifeDays of [30, 90, 180]) { + for (const importanceBoost of [0.2, 0.3, 0.5]) { + for (const frequencyBoost of [0.1, 0.2, 0.4]) { + sweeps.push({ + ...base, + minSemanticScore, + semanticWeight, + relationWeight, + decayHalfLifeDays, + importanceBoost, + frequencyBoost, + }); + } + } + } + } + } + + let best = { knobs: base, score: -1, ok: false }; + for (const k of sweeps) { + const out = await runOnce(dataset, k); + const score = out.avgNdcgAtK * 0.7 + out.avgRecallAtK * 0.3; + const better = score > best.score || (score === best.score && out.ok && !best.ok); + if (better) { + best = { knobs: k, score, ok: out.ok }; + } + } + + // eslint-disable-next-line no-console + console.log( + `sweep: best ok=${best.ok} score=${best.score.toFixed(4)} knobs=${JSON.stringify(best.knobs)}`, + ); + // eslint-disable-next-line no-console + console.log("sweep: recommended env:"); + // eslint-disable-next-line no-console + console.log(` MIN_SEMANTIC_SCORE=${best.knobs.minSemanticScore}`); + // eslint-disable-next-line no-console + console.log(` SEMANTIC_WEIGHT=${best.knobs.semanticWeight}`); + // eslint-disable-next-line no-console + console.log(` RELATION_WEIGHT=${best.knobs.relationWeight}`); + // eslint-disable-next-line no-console + console.log(` DECAY_HALF_LIFE_DAYS=${best.knobs.decayHalfLifeDays}`); + // eslint-disable-next-line no-console + console.log(` IMPORTANCE_BOOST=${best.knobs.importanceBoost}`); + // eslint-disable-next-line no-console + console.log(` FREQUENCY_BOOST=${best.knobs.frequencyBoost}`); + process.exit(best.ok ? 0 : 2); +} + +void main().catch((err) => { + // eslint-disable-next-line no-console + console.error(String(err instanceof Error ? (err.stack ?? err.message) : err)); + process.exit(1); +}); diff --git a/packages/deep-memory-server/src/eval/eval.test.ts b/packages/deep-memory-server/src/eval/eval.test.ts new file mode 100644 index 0000000000000..5ebd2b39b180c --- /dev/null +++ b/packages/deep-memory-server/src/eval/eval.test.ts @@ -0,0 +1,55 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { runEval } from "./harness.js"; +import { createRetrieverForEval } from "./stubs.js"; +import type { EvalDataset } from "./types.js"; + +const pkgRoot = path.dirname(fileURLToPath(import.meta.url)); +const datasetPath = path.resolve(pkgRoot, "../../eval/dataset.sample.json"); + +describe("eval dataset baseline", () => { + it("passes sample dataset with default knobs", async () => { + const raw = await readFile(datasetPath, "utf-8"); + const dataset = JSON.parse(raw) as EvalDataset; + const knobs = { + minSemanticScore: 0.6, + semanticWeight: 0.6, + relationWeight: 0.4, + decayHalfLifeDays: 90, + importanceBoost: 0.3, + frequencyBoost: 0.2, + }; + const cases = dataset.cases.map((c) => { + const retriever = createRetrieverForEval({ evalCase: c, knobs }); + return runEval({ retriever, cases: [c] }); + }); + const results = await Promise.all(cases); + expect(results.every((r) => r.ok)).toBe(true); + }); + + it("meets minimum quality thresholds (avg recall@k / ndcg@k)", async () => { + const raw = await readFile(datasetPath, "utf-8"); + const dataset = JSON.parse(raw) as EvalDataset; + const knobs = { + minSemanticScore: 0.6, + semanticWeight: 0.6, + relationWeight: 0.4, + decayHalfLifeDays: 90, + importanceBoost: 0.3, + frequencyBoost: 0.2, + }; + const summaries = await Promise.all( + dataset.cases.map(async (c) => { + const retriever = createRetrieverForEval({ evalCase: c, knobs }); + return await runEval({ retriever, cases: [c] }); + }), + ); + const cases = summaries.flatMap((s) => s.cases); + const avgRecallAtK = cases.reduce((sum, c) => sum + c.recallAtK, 0) / Math.max(1, cases.length); + const avgNdcgAtK = cases.reduce((sum, c) => sum + c.ndcgAtK, 0) / Math.max(1, cases.length); + expect(avgRecallAtK).toBeGreaterThanOrEqual(0.9); + expect(avgNdcgAtK).toBeGreaterThanOrEqual(0.9); + }); +}); diff --git a/packages/deep-memory-server/src/eval/harness.ts b/packages/deep-memory-server/src/eval/harness.ts new file mode 100644 index 0000000000000..4d7208e3e8862 --- /dev/null +++ b/packages/deep-memory-server/src/eval/harness.ts @@ -0,0 +1,166 @@ +import type { DeepMemoryRetriever } from "../retriever.js"; +import type { RetrieveContextResponse } from "../types.js"; +import { clamp } from "../utils.js"; +import type { EvalCase } from "./types.js"; + +export type EvalCaseResult = { + name: string; + ok: boolean; + errors: string[]; + retrievedIds: string[]; + retrievedMemoryKeys: Array; + recallAtK: number; + ndcgAtK: number; + top1Ok: boolean; + expected: { + includeIds?: string[]; + excludeIds?: string[]; + top1Id?: string; + uniqueByMemoryKey?: boolean; + }; +}; + +export function withFixedNow(isoNow: string, fn: () => T): T { + const nowMs = Date.parse(isoNow); + if (!Number.isFinite(nowMs)) { + throw new Error(`Invalid ISO time for fixed now: ${isoNow}`); + } + const prev = Date.now; + Date.now = () => nowMs; + try { + return fn(); + } finally { + Date.now = prev; + } +} + +function dcg(relevances: number[]): number { + let sum = 0; + for (let i = 0; i < relevances.length; i += 1) { + const rel = relevances[i] ?? 0; + const denom = Math.log2(i + 2); + sum += (Math.pow(2, rel) - 1) / denom; + } + return sum; +} + +function ndcgAtK(params: { predicted: string[]; relevant: Set; k: number }): number { + const k = Math.max(1, params.k); + const gains = params.predicted.slice(0, k).map((id) => (params.relevant.has(id) ? 1 : 0)); + const ideal = Array.from({ length: Math.min(k, params.relevant.size) }, () => 1); + const denom = dcg(ideal); + if (denom <= 0) { + return 1; + } + return clamp(dcg(gains) / denom, 0, 1); +} + +function recallAtK(params: { predicted: string[]; relevant: Set; k: number }): number { + const k = Math.max(1, params.k); + if (params.relevant.size === 0) { + return 1; + } + const hits = params.predicted.slice(0, k).filter((id) => params.relevant.has(id)).length; + return clamp(hits / params.relevant.size, 0, 1); +} + +function extractMemoryKeys(out: RetrieveContextResponse): Array { + return out.memories.map((m) => { + const r = m as unknown as Record; + const key = r.memory_key; + return typeof key === "string" && key.trim() ? key.trim() : undefined; + }); +} + +export async function runEvalCase(params: { + retriever: DeepMemoryRetriever; + evalCase: EvalCase; +}): Promise { + const errors: string[] = []; + const c = params.evalCase; + const relevant = new Set(c.expect.includeIds ?? []); + const k = c.query.maxMemories; + + const out = await withFixedNow(c.now, async () => { + return await params.retriever.retrieve({ + namespace: c.namespace, + userInput: c.query.userInput, + sessionId: c.query.sessionId, + maxMemories: c.query.maxMemories, + entities: c.query.entities, + topics: c.query.topics, + }); + }); + + const retrievedIds = out.memories.map((m) => m.id); + const retrievedMemoryKeys = extractMemoryKeys(out); + + const top1Ok = c.expect.top1Id ? retrievedIds[0] === c.expect.top1Id : true; + if (!top1Ok) { + errors.push(`top1 mismatch: expected=${c.expect.top1Id} actual=${retrievedIds[0] ?? ""}`); + } + + for (const id of c.expect.includeIds ?? []) { + if (!retrievedIds.includes(id)) { + errors.push(`missing expected id: ${id}`); + } + } + for (const id of c.expect.excludeIds ?? []) { + if (retrievedIds.includes(id)) { + errors.push(`unexpected id present: ${id}`); + } + } + + if (c.expect.uniqueByMemoryKey) { + const seen = new Set(); + for (const key of retrievedMemoryKeys) { + if (!key) { + continue; + } + if (seen.has(key)) { + errors.push(`duplicate memory_key returned: ${key}`); + break; + } + seen.add(key); + } + } + + return { + name: c.name, + ok: errors.length === 0, + errors, + retrievedIds, + retrievedMemoryKeys, + recallAtK: recallAtK({ predicted: retrievedIds, relevant, k }), + ndcgAtK: ndcgAtK({ predicted: retrievedIds, relevant, k }), + top1Ok, + expected: c.expect, + }; +} + +export type EvalSummary = { + ok: boolean; + cases: EvalCaseResult[]; + avgRecallAtK: number; + avgNdcgAtK: number; +}; + +export async function runEval(params: { + retriever: DeepMemoryRetriever; + cases: EvalCase[]; +}): Promise { + const results: EvalCaseResult[] = []; + for (const c of params.cases) { + results.push(await runEvalCase({ retriever: params.retriever, evalCase: c })); + } + const avgRecallAtK = + results.length === 0 ? 1 : results.reduce((s, r) => s + r.recallAtK, 0) / results.length; + const avgNdcgAtK = + results.length === 0 ? 1 : results.reduce((s, r) => s + r.ndcgAtK, 0) / results.length; + return { + ok: results.every((r) => r.ok), + cases: results, + avgRecallAtK, + avgNdcgAtK, + }; +} diff --git a/packages/deep-memory-server/src/eval/stubs.ts b/packages/deep-memory-server/src/eval/stubs.ts new file mode 100644 index 0000000000000..253190fa29585 --- /dev/null +++ b/packages/deep-memory-server/src/eval/stubs.ts @@ -0,0 +1,71 @@ +import type { DeepMemoryRetriever } from "../retriever.js"; +import { DeepMemoryRetriever as RealRetriever } from "../retriever.js"; +import type { EvalCase, EvalRelationHit, EvalSemanticHit } from "./types.js"; + +type FakeQdrant = { + search: (params: { + vector: number[]; + limit: number; + minScore: number; + namespace?: string; + }) => Promise>; +}; + +type FakeNeo4j = { + queryRelatedMemories: (params: { + namespace: string; + entities: string[]; + topics: string[]; + limit: number; + }) => Promise; +}; + +class FakeEmbedder { + async embed(_text: string): Promise { + return [0, 0, 0]; + } +} + +export function createRetrieverForEval(params: { + evalCase: EvalCase; + knobs: { + minSemanticScore: number; + semanticWeight: number; + relationWeight: number; + decayHalfLifeDays: number; + importanceBoost: number; + frequencyBoost: number; + }; +}): DeepMemoryRetriever { + const qdrant: FakeQdrant = { + search: async (p) => { + const filtered = params.evalCase.qdrantHits + .filter((h) => (!p.namespace ? true : h.id.startsWith(`${p.namespace}::`))) + .filter((h) => h.score >= p.minScore) + .slice(0, Math.max(0, p.limit)); + return filtered.map((h) => ({ id: h.id, score: h.score, payload: h.payload })); + }, + }; + const neo4j: FakeNeo4j = { + queryRelatedMemories: async (p) => { + // Namespace check is handled by the dataset itself; keep deterministic. + if (p.namespace !== params.evalCase.namespace) { + return []; + } + return params.evalCase.neo4jHits.slice(0, Math.max(0, p.limit)); + }, + }; + + return new RealRetriever({ + // We only need embed() to not throw; Qdrant is stubbed. + embedder: new FakeEmbedder() as unknown as import("../embeddings.js").EmbeddingModel, + qdrant: qdrant as unknown as import("../qdrant.js").QdrantStore, + neo4j: neo4j as unknown as import("../neo4j.js").Neo4jStore, + minSemanticScore: params.knobs.minSemanticScore, + semanticWeight: params.knobs.semanticWeight, + relationWeight: params.knobs.relationWeight, + decayHalfLifeDays: params.knobs.decayHalfLifeDays, + importanceBoost: params.knobs.importanceBoost, + frequencyBoost: params.knobs.frequencyBoost, + }); +} diff --git a/packages/deep-memory-server/src/eval/types.ts b/packages/deep-memory-server/src/eval/types.ts new file mode 100644 index 0000000000000..de55aa689b96c --- /dev/null +++ b/packages/deep-memory-server/src/eval/types.ts @@ -0,0 +1,63 @@ +export type EvalSemanticHit = { + id: string; + score: number; + payload: { + content: string; + importance?: number; + frequency?: number; + created_at: string; + updated_at?: string; + kind?: string; + memory_key?: string; + subject?: string; + expires_at?: string; + confidence?: number; + }; +}; + +export type EvalRelationHit = { + id: string; + content: string; + importance: number; + frequency: number; + lastSeenAt: string; + relationScore: number; + kind?: string; + memoryKey?: string; + subject?: string; + expiresAt?: string; + confidence?: number; +}; + +export type EvalCase = { + name: string; + namespace: string; + now: string; + query: { + userInput: string; + sessionId: string; + entities: string[]; + topics: string[]; + maxMemories: number; + }; + qdrantHits: EvalSemanticHit[]; + neo4jHits: EvalRelationHit[]; + expect: { + /** + * Expected to appear in the final retrieved list (any rank). + * Use ids (the merged id namespace::mem_...). + */ + includeIds?: string[]; + /** Expected to NOT appear (expired, conflict-resolved, below threshold). */ + excludeIds?: string[]; + /** If provided, expected top-1 id. */ + top1Id?: string; + /** If provided, enforce that only one memory per memory_key is returned. */ + uniqueByMemoryKey?: boolean; + }; +}; + +export type EvalDataset = { + version: number; + cases: EvalCase[]; +}; diff --git a/packages/deep-memory-server/src/gc-expired.ts b/packages/deep-memory-server/src/gc-expired.ts new file mode 100644 index 0000000000000..4eb46fe9f087a --- /dev/null +++ b/packages/deep-memory-server/src/gc-expired.ts @@ -0,0 +1,250 @@ +import fs from "node:fs/promises"; +import process from "node:process"; +import { loadConfig } from "./config.js"; +import { createLogger } from "./logger.js"; +import { Neo4jStore } from "./neo4j.js"; +import { QdrantStore } from "./qdrant.js"; +import { DEEPMEM_SCHEMA_VERSION } from "./schema.js"; + +type Args = { + namespace?: string; + allNamespaces: boolean; + batchSize: number; + deleteBatch: number; + maxItems?: number; + dryRun: boolean; + targetCollection?: string; + outFile?: string; +}; + +function parseArgs(argv: string[]): Args { + const out: Args = { allNamespaces: false, batchSize: 200, deleteBatch: 100, dryRun: false }; + for (let i = 0; i < argv.length; i += 1) { + const t = argv[i]?.trim(); + if (!t) { + continue; + } + if (t === "--namespace" || t === "-n") { + out.namespace = argv[i + 1]?.trim(); + i += 1; + continue; + } + if (t === "--all-namespaces" || t === "--all") { + out.allNamespaces = true; + continue; + } + if (t === "--batch-size") { + const v = Number(argv[i + 1]); + if (Number.isFinite(v) && v > 0) { + out.batchSize = Math.max(1, Math.min(1000, Math.floor(v))); + } + i += 1; + continue; + } + if (t === "--delete-batch") { + const v = Number(argv[i + 1]); + if (Number.isFinite(v) && v > 0) { + out.deleteBatch = Math.max(1, Math.min(1000, Math.floor(v))); + } + i += 1; + continue; + } + if (t === "--max-items") { + const v = Number(argv[i + 1]); + if (Number.isFinite(v) && v > 0) { + out.maxItems = Math.max(1, Math.floor(v)); + } + i += 1; + continue; + } + if (t === "--target-collection" || t === "--collection") { + out.targetCollection = argv[i + 1]?.trim(); + i += 1; + continue; + } + if (t === "--out") { + const v = argv[i + 1]?.trim(); + if (v) { + out.outFile = v; + } + i += 1; + continue; + } + if (t === "--dry-run") { + out.dryRun = true; + continue; + } + if (t === "--help" || t === "-h") { + // eslint-disable-next-line no-console + console.log(` +Usage: pnpm --dir packages/deep-memory-server gc-expired -- --namespace [options] + +Options: + --namespace, -n GC expired memories in a single namespace + --all-namespaces, --all GC expired memories in all namespaces + --batch-size Scan page size (default: 200) + --delete-batch Delete batch size (default: 100) + --max-items Stop after scanning N items + --target-collection Qdrant collection (default: QDRANT_COLLECTION) + --out Write JSON report to file + --dry-run Do not delete, only report + --help, -h Show help +`); + process.exit(0); + } + } + if (!out.allNamespaces) { + out.namespace = out.namespace?.trim() || undefined; + if (!out.namespace) { + throw new Error("missing --namespace (or use --all-namespaces)"); + } + } + return out; +} + +function isExpired(expiresAt: string | undefined, nowMs: number): boolean { + const t = (expiresAt ?? "").trim(); + if (!t) { + return false; + } + const ms = Date.parse(t); + return Number.isFinite(ms) && ms > 0 && ms <= nowMs; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const cfg = loadConfig(); + const log = createLogger(cfg.LOG_LEVEL); + const targetCollection = (args.targetCollection ?? cfg.QDRANT_COLLECTION).trim(); + if (!targetCollection) { + throw new Error("empty target collection"); + } + + const neo4j = new Neo4jStore({ + uri: cfg.NEO4J_URI, + user: cfg.NEO4J_USER, + password: cfg.NEO4J_PASSWORD, + }); + const qdrant = new QdrantStore({ + url: cfg.QDRANT_URL, + apiKey: cfg.QDRANT_API_KEY, + collection: targetCollection, + dims: cfg.VECTOR_DIMS, + }); + await qdrant.schemaStatus({ mode: "validate", expectedVersion: DEEPMEM_SCHEMA_VERSION }); + + const namespaces = args.allNamespaces + ? await neo4j.listNamespaces({ limit: 10_000 }) + : [args.namespace!]; + + const nowMs = Date.now(); + const report: { + ok: boolean; + dryRun: boolean; + targetCollection: string; + scanned: number; + expiredFound: number; + neo4jDeleted: number; + qdrantDeleted: number; + failedBatches: number; + finishedAt: string; + } = { + ok: true, + dryRun: args.dryRun, + targetCollection, + scanned: 0, + expiredFound: 0, + neo4jDeleted: 0, + qdrantDeleted: 0, + failedBatches: 0, + finishedAt: "", + }; + + try { + for (const ns of namespaces) { + let afterId: string | undefined; + const batchIds: string[] = []; + while (true) { + const page = await neo4j.scanMemories({ namespace: ns, afterId, limit: args.batchSize }); + if (page.length === 0) { + break; + } + for (const m of page) { + report.scanned += 1; + afterId = m.id; + if (args.maxItems && report.scanned >= args.maxItems) { + break; + } + if (isExpired(m.expiresAt, nowMs)) { + report.expiredFound += 1; + batchIds.push(m.id); + } + if (batchIds.length >= args.deleteBatch) { + if (!args.dryRun) { + try { + report.qdrantDeleted += await qdrant.deleteByIds({ ids: batchIds }); + } catch { + report.failedBatches += 1; + report.ok = false; + } + try { + report.neo4jDeleted += await neo4j.deleteMemoriesByIds({ + namespace: ns, + ids: batchIds, + }); + } catch { + report.failedBatches += 1; + report.ok = false; + } + } + batchIds.length = 0; + } + } + if (args.maxItems && report.scanned >= args.maxItems) { + break; + } + } + if (batchIds.length > 0) { + if (!args.dryRun) { + try { + report.qdrantDeleted += await qdrant.deleteByIds({ ids: batchIds }); + } catch { + report.failedBatches += 1; + report.ok = false; + } + try { + report.neo4jDeleted += await neo4j.deleteMemoriesByIds({ + namespace: ns, + ids: batchIds, + }); + } catch { + report.failedBatches += 1; + report.ok = false; + } + } + } + log.info( + { namespace: ns, scanned: report.scanned, expiredFound: report.expiredFound }, + "gc-expired namespace progress", + ); + } + } finally { + await neo4j.close().catch(() => {}); + } + + report.finishedAt = new Date().toISOString(); + if (args.outFile?.trim()) { + await fs.writeFile(args.outFile.trim(), `${JSON.stringify(report, null, 2)}\n`, "utf8"); + } + // eslint-disable-next-line no-console + console.log( + `gc-expired: ok=${report.ok} dryRun=${report.dryRun} scanned=${report.scanned} expired=${report.expiredFound} neo4jDeleted=${report.neo4jDeleted} qdrantDeleted=${report.qdrantDeleted}`, + ); + process.exit(report.ok ? 0 : 2); +} + +void main().catch((err) => { + // eslint-disable-next-line no-console + console.error(String(err instanceof Error ? (err.stack ?? err.message) : err)); + process.exit(1); +}); diff --git a/packages/deep-memory-server/src/idempotency.test.ts b/packages/deep-memory-server/src/idempotency.test.ts new file mode 100644 index 0000000000000..5c7df8d49f4eb --- /dev/null +++ b/packages/deep-memory-server/src/idempotency.test.ts @@ -0,0 +1,75 @@ +import crypto from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { DeepMemoryUpdater } from "./updater.js"; + +describe("DeepMemoryUpdater idempotency", () => { + it("skips when transcriptHash already ingested", async () => { + const analyzer = { + analyze: () => ({ + entities: [], + topics: [], + events: [], + drafts: [], + filtered: { added: 0, filtered: 0 }, + }), + }; + const embedder = { embed: async () => [0, 0, 0] }; + const qdrant = { + search: async () => [], + getMemory: async () => null, + upsertMemory: async () => {}, + }; + + const messages = [{ role: "user", content: "hello" }]; + const hash = crypto.createHash("sha256").update(JSON.stringify(messages)).digest("hex"); + + let calls = 0; + const neo4j = { + upsertSession: async () => {}, + getSessionIngestMeta: async () => { + calls += 1; + // First call: no meta. Second call: meta exists and should cause skip. + if (calls === 1) { + return {}; + } + return { transcriptHash: hash }; + }, + setSessionIngestMeta: async () => {}, + upsertTopic: async () => {}, + linkSessionTopic: async () => {}, + upsertEntity: async () => {}, + linkTopicEntity: async () => {}, + upsertEvent: async () => {}, + eventId: () => "e1", + linkSessionEvent: async () => {}, + linkEventTopic: async () => {}, + linkEventEntity: async () => {}, + upsertMemory: async () => {}, + linkMemoryTopic: async () => {}, + linkMemoryEntity: async () => {}, + linkMemoryRelated: async () => {}, + }; + + const updater = new DeepMemoryUpdater({ + analyzer: analyzer as unknown as never, + embedder: embedder as unknown as never, + qdrant: qdrant as unknown as never, + neo4j: neo4j as unknown as never, + minSemanticScore: 0.6, + importanceThreshold: 0.5, + maxMemoriesPerUpdate: 20, + dedupeScore: 0.92, + relatedTopK: 0, + sensitiveFilterEnabled: true, + }); + + // First update: will set meta (best-effort), but our stub doesn't persist it. + const a = await updater.update({ namespace: "default", sessionId: "s1", messages }); + expect(a.status).toBe("processed"); + + // Second update: should skip when meta says the hash is already ingested. + const b = await updater.update({ namespace: "default", sessionId: "s1", messages }); + expect(b.status).toBe("skipped"); + expect(calls).toBeGreaterThanOrEqual(2); + }); +}); diff --git a/packages/deep-memory-server/src/importance.test.ts b/packages/deep-memory-server/src/importance.test.ts new file mode 100644 index 0000000000000..73a7c833e900a --- /dev/null +++ b/packages/deep-memory-server/src/importance.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { computeImportance } from "./importance.js"; + +describe("computeImportance", () => { + it("returns 0 for empty input", () => { + expect(computeImportance(null)).toBe(0); + expect(computeImportance(undefined)).toBe(0); + }); + + it("clamps to [0,1] and respects weights", () => { + const high = computeImportance({ + frequency: 10, + novelty: 1, + user_intent: 1, + length: 2000, + }); + expect(high).toBeGreaterThan(0.8); + + const low = computeImportance({ + frequency: 0, + novelty: 0, + user_intent: 0, + length: 0, + }); + expect(low).toBe(0); + }); +}); diff --git a/packages/deep-memory-server/src/importance.ts b/packages/deep-memory-server/src/importance.ts new file mode 100644 index 0000000000000..28fa791ed1380 --- /dev/null +++ b/packages/deep-memory-server/src/importance.ts @@ -0,0 +1,32 @@ +import { clamp } from "./utils.js"; + +export type ImportanceSignals = { + frequency: number; + novelty: number; // 0..1 + user_intent: number; // 0..1 + length: number; // approximate turn length +}; + +/** + * Requirement weights: + * - frequency 30% + * - novelty 25% + * - user_intent 30% + * - length 15% + */ +export function computeImportance(input: Partial | null | undefined): number { + if (!input) { + return 0; + } + const frequency = clamp(Number(input.frequency ?? 0), 0, 100); + const novelty = clamp(Number(input.novelty ?? 0), 0, 1); + const userIntent = clamp(Number(input.user_intent ?? 0), 0, 1); + const length = clamp(Number(input.length ?? 0), 0, 10_000); + + // Normalize frequency/length to 0..1 with simple saturating curves. + const freqScore = clamp(frequency / 10, 0, 1); + const lenScore = clamp(length / 2000, 0, 1); + + const score = 0.3 * freqScore + 0.25 * novelty + 0.3 * userIntent + 0.15 * lenScore; + return clamp(score, 0, 1); +} diff --git a/packages/deep-memory-server/src/index.ts b/packages/deep-memory-server/src/index.ts new file mode 100644 index 0000000000000..4b966579de7fc --- /dev/null +++ b/packages/deep-memory-server/src/index.ts @@ -0,0 +1,174 @@ +import { serve } from "@hono/node-server"; +import { LRUCache } from "lru-cache"; +import { SessionAnalyzer } from "./analyzer.js"; +import { createApi } from "./api.js"; +import { loadConfig } from "./config.js"; +import { DurableForgetQueue } from "./durable-forget-queue.js"; +import { DurableUpdateQueue } from "./durable-update-queue.js"; +import { EmbeddingModel } from "./embeddings.js"; +import { createLogger } from "./logger.js"; +import { createMetrics } from "./metrics.js"; +import { Neo4jStore } from "./neo4j.js"; +import { QdrantStore } from "./qdrant.js"; +import { DeepMemoryRetriever } from "./retriever.js"; +import { createSensitiveFilter } from "./safety.js"; +import { DEEPMEM_SCHEMA_VERSION } from "./schema.js"; +import type { RetrieveContextResponse } from "./types.js"; +import { DeepMemoryUpdater } from "./updater.js"; + +async function main() { + const cfg = loadConfig(); + const log = createLogger(cfg.LOG_LEVEL); + + const embedder = new EmbeddingModel({ modelId: cfg.EMBEDDING_MODEL, dims: cfg.VECTOR_DIMS }); + const qdrant = new QdrantStore({ + url: cfg.QDRANT_URL, + apiKey: cfg.QDRANT_API_KEY, + collection: cfg.QDRANT_COLLECTION, + dims: cfg.VECTOR_DIMS, + }); + const neo4j = new Neo4jStore({ + uri: cfg.NEO4J_URI, + user: cfg.NEO4J_USER, + password: cfg.NEO4J_PASSWORD, + }); + + // Init (best-effort): if either backend is down, server still starts and will degrade. + try { + const status = await qdrant.schemaStatus({ + mode: cfg.MIGRATIONS_MODE, + expectedVersion: DEEPMEM_SCHEMA_VERSION, + }); + if (!status.ok) { + log.warn({ status }, "qdrant schema not ready"); + if (cfg.MIGRATIONS_STRICT) { + throw new Error("qdrant schema not ready (strict)"); + } + } else { + log.info({ status }, "qdrant schema ready"); + } + } catch (err) { + log.warn({ err: String(err) }, "qdrant unavailable at startup (will degrade)"); + } + try { + const status = await neo4j.schemaStatus({ + mode: cfg.MIGRATIONS_MODE, + expectedVersion: DEEPMEM_SCHEMA_VERSION, + }); + if (!status.ok) { + log.warn({ status }, "neo4j schema not ready"); + if (cfg.MIGRATIONS_STRICT) { + throw new Error("neo4j schema not ready (strict)"); + } + } else { + log.info({ status }, "neo4j schema ready"); + } + } catch (err) { + log.warn({ err: String(err) }, "neo4j unavailable at startup (will degrade)"); + } + + const retriever = new DeepMemoryRetriever({ + embedder, + qdrant, + neo4j, + minSemanticScore: cfg.MIN_SEMANTIC_SCORE, + semanticWeight: cfg.SEMANTIC_WEIGHT, + relationWeight: cfg.RELATION_WEIGHT, + decayHalfLifeDays: cfg.DECAY_HALF_LIFE_DAYS, + importanceBoost: cfg.IMPORTANCE_BOOST, + frequencyBoost: cfg.FREQUENCY_BOOST, + }); + const updater = new DeepMemoryUpdater({ + analyzer: new SessionAnalyzer(), + embedder, + qdrant, + neo4j, + minSemanticScore: cfg.MIN_SEMANTIC_SCORE, + importanceThreshold: cfg.IMPORTANCE_THRESHOLD, + maxMemoriesPerUpdate: cfg.MAX_MEMORIES_PER_UPDATE, + dedupeScore: cfg.DEDUPE_SCORE, + relatedTopK: cfg.RELATED_TOPK, + sensitiveFilterEnabled: cfg.SENSITIVE_FILTER_ENABLED, + sensitiveFilter: createSensitiveFilter(cfg), + }); + + const updateQueue = new DurableUpdateQueue({ + log, + updater, + concurrency: cfg.UPDATE_CONCURRENCY, + namespaceConcurrency: cfg.NAMESPACE_UPDATE_CONCURRENCY, + dir: cfg.QUEUE_DIR, + maxAttempts: cfg.QUEUE_MAX_ATTEMPTS, + retryBaseMs: cfg.QUEUE_RETRY_BASE_MS, + retryMaxMs: cfg.QUEUE_RETRY_MAX_MS, + keepDone: cfg.QUEUE_KEEP_DONE, + retentionDays: cfg.QUEUE_RETENTION_DAYS, + maxTaskBytes: cfg.QUEUE_MAX_TASK_BYTES, + }); + await updateQueue.init(); + + const forgetQueue = new DurableForgetQueue({ + log, + qdrant, + neo4j, + updateQueue, + concurrency: 1, + dir: `${cfg.QUEUE_DIR.replace(/\/+$/, "")}/forget`, + maxAttempts: cfg.QUEUE_MAX_ATTEMPTS, + retryBaseMs: cfg.QUEUE_RETRY_BASE_MS, + retryMaxMs: cfg.QUEUE_RETRY_MAX_MS, + keepDone: cfg.QUEUE_KEEP_DONE, + retentionDays: cfg.QUEUE_RETENTION_DAYS, + }); + await forgetQueue.init(); + + // Retrieve cache (server-side): best-effort; client-side cache exists too. + const retrieveCache = new LRUCache>({ + max: cfg.RETRIEVE_CACHE_MAX, + ttl: cfg.RETRIEVE_CACHE_TTL_MS, + }); + + const baseRetrieve = retriever.retrieve.bind(retriever); + // Avoid re-embedding the same input across concurrent callers. + retriever.retrieve = async (params: Parameters[0]) => { + const key = `${params.namespace}::${params.sessionId}::${params.maxMemories}::${params.userInput.trim()}`; + const cached = retrieveCache.get(key); + if (cached) { + return await cached; + } + const promise = baseRetrieve(params); + retrieveCache.set(key, promise); + return await promise; + }; + + const app = createApi({ + cfg, + log, + retriever, + updater, + qdrant, + neo4j, + queue: updateQueue, + forgetQueue, + metrics: createMetrics(), + }); + + const server = serve({ fetch: app.fetch, port: cfg.PORT, hostname: cfg.HOST }); + log.info({ host: cfg.HOST, port: cfg.PORT }, "deep-memory-server listening"); + + const shutdown = async () => { + try { + server.close(); + } catch {} + try { + await neo4j.close(); + } catch {} + try { + updateQueue.stop(); + } catch {} + }; + process.on("SIGINT", () => void shutdown().finally(() => process.exit(0))); + process.on("SIGTERM", () => void shutdown().finally(() => process.exit(0))); +} + +void main(); diff --git a/packages/deep-memory-server/src/logger.ts b/packages/deep-memory-server/src/logger.ts new file mode 100644 index 0000000000000..8c666a74a08a9 --- /dev/null +++ b/packages/deep-memory-server/src/logger.ts @@ -0,0 +1,20 @@ +import pino from "pino"; + +export function createLogger(level: string) { + return pino({ + level, + redact: { + paths: [ + "req.headers.authorization", + "req.headers.cookie", + "req.headers['x-api-key']", + "body.password", + "body.apiKey", + "body.token", + "NEO4J_PASSWORD", + "QDRANT_API_KEY", + ], + remove: true, + }, + }); +} diff --git a/packages/deep-memory-server/src/metrics.ts b/packages/deep-memory-server/src/metrics.ts new file mode 100644 index 0000000000000..b2e19d35f8ef9 --- /dev/null +++ b/packages/deep-memory-server/src/metrics.ts @@ -0,0 +1,80 @@ +import { Registry, collectDefaultMetrics, Counter, Gauge, Histogram } from "prom-client"; + +export type DeepMemoryMetrics = ReturnType; + +export function createMetrics() { + const registry = new Registry(); + collectDefaultMetrics({ register: registry }); + + const httpRequestsTotal = new Counter({ + name: "deep_memory_http_requests_total", + help: "Total HTTP requests handled by deep-memory-server", + registers: [registry], + labelNames: ["route", "method", "status"] as const, + }); + + const httpRequestDurationSeconds = new Histogram({ + name: "deep_memory_http_request_duration_seconds", + help: "HTTP request duration in seconds (deep-memory-server)", + registers: [registry], + labelNames: ["route", "method", "status"] as const, + buckets: [0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10], + }); + + const retrieveReturnedMemoriesTotal = new Counter({ + name: "deep_memory_retrieve_returned_memories_total", + help: "Total number of memories returned from /retrieve_context", + registers: [registry], + labelNames: ["status"] as const, + }); + + const updateMemoriesAddedTotal = new Counter({ + name: "deep_memory_update_memories_added_total", + help: "Total number of memories added by /update_memory_index (sync path)", + registers: [registry], + labelNames: ["status"] as const, + }); + + const updateMemoriesFilteredTotal = new Counter({ + name: "deep_memory_update_memories_filtered_total", + help: "Total number of memories filtered by /update_memory_index (sync path)", + registers: [registry], + labelNames: ["status"] as const, + }); + + const forgetDeletedTotal = new Counter({ + name: "deep_memory_forget_deleted_total", + help: "Total deleted memories reported by /forget (best-effort)", + registers: [registry], + labelNames: ["status"] as const, + }); + + const queuePending = new Gauge({ + name: "deep_memory_queue_pending", + help: "Approx pending update tasks (latest-by-key)", + registers: [registry], + }); + const queueActive = new Gauge({ + name: "deep_memory_queue_active", + help: "Active update tasks currently running", + registers: [registry], + }); + const queueInflightKeys = new Gauge({ + name: "deep_memory_queue_inflight_keys", + help: "Number of keys currently in-flight (mutual exclusion set size)", + registers: [registry], + }); + + return { + registry, + httpRequestsTotal, + httpRequestDurationSeconds, + retrieveReturnedMemoriesTotal, + updateMemoriesAddedTotal, + updateMemoriesFilteredTotal, + forgetDeletedTotal, + queuePending, + queueActive, + queueInflightKeys, + }; +} diff --git a/packages/deep-memory-server/src/neo4j.scan.test.ts b/packages/deep-memory-server/src/neo4j.scan.test.ts new file mode 100644 index 0000000000000..3119e748ffb98 --- /dev/null +++ b/packages/deep-memory-server/src/neo4j.scan.test.ts @@ -0,0 +1,49 @@ +import type { Driver } from "neo4j-driver"; +import { describe, expect, it } from "vitest"; +import { Neo4jStore } from "./neo4j.js"; + +function createStoreWithRun(result: unknown[]) { + const store = new Neo4jStore({ uri: "bolt://x", user: "u", password: "p" }); + const fake: Partial = { + session: () => + ({ + run: async () => ({ + records: result.map((row) => ({ + get: (k: string) => (row as Record)[k], + })), + }), + close: async () => {}, + }) as unknown, + }; + // Inject fake driver (tests only). + (store as unknown as { driver: Driver }).driver = fake as Driver; + return store; +} + +describe("Neo4jStore.scanMemories", () => { + it("parses sessionId from namespaced session node id", async () => { + const store = createStoreWithRun([ + { + id: "ns1::mem_x", + namespace: "ns1", + content: "hello", + createdAt: "2026-01-01T00:00:00.000Z", + kind: "fact", + memoryKey: "", + subject: "", + expiresAt: "", + confidence: 0.5, + importance: 0.9, + frequency: 2, + sessionNodeId: "ns1::session::sess-123", + topics: ["t1"], + entities: [{ name: "e1", type: "person" }], + }, + ]); + + const out = await store.scanMemories({ namespace: "ns1", limit: 10 }); + expect(out[0]?.sessionId).toBe("sess-123"); + expect(out[0]?.topics).toEqual(["t1"]); + expect(out[0]?.entities[0]?.name).toBe("e1"); + }); +}); diff --git a/packages/deep-memory-server/src/neo4j.ts b/packages/deep-memory-server/src/neo4j.ts new file mode 100644 index 0000000000000..bf360fa156a14 --- /dev/null +++ b/packages/deep-memory-server/src/neo4j.ts @@ -0,0 +1,947 @@ +import neo4j, { type Driver } from "neo4j-driver"; +import type { SchemaCheckResult } from "./schema.js"; +import type { CandidateMemory, ExtractedEntity, ExtractedEvent, ExtractedTopic } from "./types.js"; + +export class Neo4jStore { + private readonly driver: Driver; + + constructor(params: { uri: string; user: string; password: string }) { + this.driver = neo4j.driver(params.uri, neo4j.auth.basic(params.user, params.password), { + // Keep defaults; the caller controls availability via retry/fallback. + }); + } + + async close(): Promise { + await this.driver.close(); + } + + async healthCheck(): Promise<{ ok: true } | { ok: false; error: string }> { + try { + // verifyConnectivity is the lightest built-in probe. + await this.driver.verifyConnectivity(); + return { ok: true }; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : String(err) }; + } + } + + private async listConstraints(): Promise< + Array<{ + name: string; + type: string; + labelsOrTypes: string[]; + properties: string[]; + }> + > { + const session = this.driver.session(); + try { + const res = await session.run( + "SHOW CONSTRAINTS YIELD name, type, labelsOrTypes, properties RETURN name, type, labelsOrTypes, properties", + ); + return res.records.map((r) => { + const labels = r.get("labelsOrTypes") as unknown; + const props = r.get("properties") as unknown; + return { + name: String(r.get("name") ?? ""), + type: String(r.get("type") ?? ""), + labelsOrTypes: Array.isArray(labels) ? labels.map((x) => String(x)) : [], + properties: Array.isArray(props) ? props.map((x) => String(x)) : [], + }; + }); + } finally { + await session.close(); + } + } + + private async listIndexes(): Promise< + Array<{ + name: string; + type: string; + labelsOrTypes: string[]; + properties: string[]; + }> + > { + const session = this.driver.session(); + try { + const res = await session.run( + "SHOW INDEXES YIELD name, type, labelsOrTypes, properties RETURN name, type, labelsOrTypes, properties", + ); + return res.records.map((r) => { + const labels = r.get("labelsOrTypes") as unknown; + const props = r.get("properties") as unknown; + return { + name: String(r.get("name") ?? ""), + type: String(r.get("type") ?? ""), + labelsOrTypes: Array.isArray(labels) ? labels.map((x) => String(x)) : [], + properties: Array.isArray(props) ? props.map((x) => String(x)) : [], + }; + }); + } finally { + await session.close(); + } + } + + private hasUniquenessConstraint( + constraints: Awaited>, + label: string, + property: string, + ): boolean { + return constraints.some((c) => { + const t = c.type.toUpperCase(); + return ( + t.includes("UNIQUENESS") && + c.labelsOrTypes.includes(label) && + c.properties.length === 1 && + c.properties[0] === property + ); + }); + } + + private hasBtreeIndex( + indexes: Awaited>, + label: string, + property: string, + ): boolean { + return indexes.some((idx) => { + const t = idx.type.toUpperCase(); + return ( + (t.includes("BTREE") || t.includes("RANGE") || t.includes("TEXT")) && + idx.labelsOrTypes.includes(label) && + idx.properties.length === 1 && + idx.properties[0] === property + ); + }); + } + + private async getSchemaVersion(): Promise { + const session = this.driver.session(); + try { + const res = await session.run( + "MATCH (m:DeepMemMeta {id: 'schema'}) RETURN m.schema_version AS v LIMIT 1", + ); + const row = res.records[0]; + if (!row) { + return undefined; + } + const v = row.get("v") as unknown; + return typeof v === "number" && Number.isFinite(v) ? v : undefined; + } finally { + await session.close(); + } + } + + private async setSchemaVersion(v: number): Promise { + const session = this.driver.session(); + try { + await session.run( + "MERGE (m:DeepMemMeta {id: 'schema'}) SET m.schema_version = $v, m.updated_at = datetime()", + { v }, + ); + } finally { + await session.close(); + } + } + + private async runSchemaStatement(cypher: string): Promise { + const session = this.driver.session(); + try { + await session.run(cypher); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + // Neo4j will error if an equivalent constraint exists under another name. + if (msg.toLowerCase().includes("already exists")) { + return; + } + throw err; + } finally { + await session.close(); + } + } + + async schemaStatus(params: { + mode: SchemaCheckResult["mode"]; + expectedVersion: number; + }): Promise { + const actions: string[] = []; + const warnings: string[] = []; + try { + const currentVersion = await this.getSchemaVersion(); + if (params.mode === "off") { + return { + ok: true, + mode: params.mode, + expectedVersion: params.expectedVersion, + currentVersion, + }; + } + + const constraints = await this.listConstraints(); + const indexes = await this.listIndexes(); + + const requiredUnique: Array<{ label: string; property: string; name: string }> = [ + { label: "Session", property: "id", name: "deepmem_session_id" }, + { label: "Memory", property: "id", name: "deepmem_memory_id" }, + { label: "Topic", property: "id", name: "deepmem_topic_id" }, + { label: "Entity", property: "id", name: "deepmem_entity_id" }, + { label: "Event", property: "id", name: "deepmem_event_id" }, + ]; + + const requiredIndexes: Array<{ label: string; property: string; name: string }> = [ + { label: "Entity", property: "name", name: "deepmem_entity_name" }, + { label: "Topic", property: "name", name: "deepmem_topic_name" }, + { label: "Memory", property: "memory_key", name: "deepmem_memory_key" }, + ]; + + const missingUniques = requiredUnique.filter( + (r) => !this.hasUniquenessConstraint(constraints, r.label, r.property), + ); + const missingIndexes = requiredIndexes.filter( + (r) => !this.hasBtreeIndex(indexes, r.label, r.property), + ); + + if (params.mode === "apply") { + for (const c of missingUniques) { + await this.runSchemaStatement( + `CREATE CONSTRAINT ${c.name} IF NOT EXISTS FOR (n:${c.label}) REQUIRE n.${c.property} IS UNIQUE`, + ); + actions.push(`created constraint ${c.name}`); + } + for (const idx of missingIndexes) { + await this.runSchemaStatement( + `CREATE INDEX ${idx.name} IF NOT EXISTS FOR (n:${idx.label}) ON (n.${idx.property})`, + ); + actions.push(`created index ${idx.name}`); + } + await this.setSchemaVersion(params.expectedVersion); + } else { + for (const c of missingUniques) { + warnings.push(`missing uniqueness constraint on :${c.label}(${c.property})`); + } + for (const idx of missingIndexes) { + warnings.push(`missing index on :${idx.label}(${idx.property})`); + } + } + + const ok = + missingUniques.length === 0 && + missingIndexes.length === 0 && + (currentVersion == null || currentVersion <= params.expectedVersion); + + return { + ok: params.mode === "apply" ? true : ok, + mode: params.mode, + expectedVersion: params.expectedVersion, + currentVersion, + actions: actions.length ? actions : undefined, + warnings: warnings.length ? warnings : undefined, + }; + } catch (err) { + return { + ok: false, + mode: params.mode, + expectedVersion: params.expectedVersion, + error: err instanceof Error ? err.message : String(err), + actions: actions.length ? actions : undefined, + warnings: warnings.length ? warnings : undefined, + }; + } + } + + async ensureSchema(): Promise { + // Back-compat: keep behavior (create missing constraints/indexes). + await this.schemaStatus({ mode: "apply", expectedVersion: 0 }); + } + + private prefix(namespace: string): string { + return `${namespace}::`; + } + + private parseSessionIdFromNodeId(nodeId: string): string | undefined { + // Format: `${namespace}::session::` + const idx = nodeId.indexOf("session::"); + if (idx < 0) { + return undefined; + } + const out = nodeId.slice(idx + "session::".length); + return out.trim() || undefined; + } + + private sessionNodeId(namespace: string, sessionId: string): string { + return `${this.prefix(namespace)}session::${sessionId}`; + } + + private topicNodeId(namespace: string, topicName: string): string { + return `${this.prefix(namespace)}topic::${topicName}`; + } + + private entityNodeId(namespace: string, type: string, name: string): string { + return `${this.prefix(namespace)}entity::${type}::${name}`; + } + + private eventNodeId(namespace: string, event: ExtractedEvent): string { + return `${this.prefix(namespace)}event::${event.type}::${event.timestamp}::${event.summary}`.slice( + 0, + 240, + ); + } + + async upsertSession(params: { + namespace: string; + sessionId: string; + startTime?: string; + endTime?: string; + summary?: string; + }) { + const session = this.driver.session(); + try { + await session.run( + `MERGE (s:Session {id: $id}) + ON CREATE SET s.start_time = coalesce($start_time, datetime()) + SET s.end_time = $end_time, + s.summary = $summary, + s.namespace = $ns`, + { + id: this.sessionNodeId(params.namespace, params.sessionId), + ns: params.namespace, + start_time: params.startTime ?? null, + end_time: params.endTime ?? null, + summary: params.summary ?? null, + }, + ); + } finally { + await session.close(); + } + } + + async getSessionIngestMeta(params: { namespace: string; sessionId: string }): Promise<{ + transcriptHash?: string; + messageCount?: number; + }> { + const session = this.driver.session(); + try { + const res = await session.run( + `MATCH (s:Session {id: $id}) + RETURN s.last_transcript_hash AS hash, s.last_message_count AS count`, + { id: this.sessionNodeId(params.namespace, params.sessionId) }, + ); + const row = res.records[0]; + if (!row) { + return {}; + } + const hash = row.get("hash") as string | null | undefined; + const count = row.get("count") as number | null | undefined; + return { + transcriptHash: typeof hash === "string" && hash.length > 0 ? hash : undefined, + messageCount: typeof count === "number" && Number.isFinite(count) ? count : undefined, + }; + } finally { + await session.close(); + } + } + + async setSessionIngestMeta(params: { + namespace: string; + sessionId: string; + transcriptHash: string; + messageCount: number; + }): Promise { + const session = this.driver.session(); + try { + await session.run( + `MATCH (s:Session {id: $id}) + SET s.last_transcript_hash = $hash, + s.last_message_count = $count, + s.last_ingested_at = datetime()`, + { + id: this.sessionNodeId(params.namespace, params.sessionId), + hash: params.transcriptHash, + count: params.messageCount, + }, + ); + } finally { + await session.close(); + } + } + + async upsertTopic(params: { namespace: string; topic: ExtractedTopic }): Promise { + const session = this.driver.session(); + try { + await session.run( + `MERGE (t:Topic {id: $id}) + ON CREATE SET t.name = $name, t.frequency = 0, t.importance = 0, t.namespace = $ns + SET t.frequency = coalesce(t.frequency, 0) + $frequency, + t.importance = greatest(coalesce(t.importance, 0), $importance)`, + { + id: this.topicNodeId(params.namespace, params.topic.name), + ns: params.namespace, + name: params.topic.name, + frequency: params.topic.frequency, + importance: params.topic.importance, + }, + ); + } finally { + await session.close(); + } + } + + async upsertEntity(params: { namespace: string; entity: ExtractedEntity }): Promise { + const session = this.driver.session(); + try { + await session.run( + `MERGE (e:Entity {id: $id}) + ON CREATE SET e.name = $name, e.type = $type, e.frequency = 0, e.namespace = $ns + SET e.frequency = coalesce(e.frequency, 0) + $frequency`, + { + id: this.entityNodeId(params.namespace, params.entity.type, params.entity.name), + ns: params.namespace, + name: params.entity.name, + type: params.entity.type, + frequency: params.entity.frequency, + }, + ); + } finally { + await session.close(); + } + } + + async upsertEvent(params: { namespace: string; event: ExtractedEvent }): Promise { + const session = this.driver.session(); + try { + await session.run( + `MERGE (e:Event {id: $id}) + ON CREATE SET e.type = $type, e.summary = $summary, e.timestamp = datetime($ts), e.namespace = $ns + SET e.type = $type, e.summary = $summary`, + { + id: this.eventNodeId(params.namespace, params.event), + ns: params.namespace, + type: params.event.type, + summary: params.event.summary, + ts: params.event.timestamp, + }, + ); + } finally { + await session.close(); + } + } + + eventId(params: { namespace: string; event: ExtractedEvent }): string { + return this.eventNodeId(params.namespace, params.event); + } + + async linkSessionEvent(params: { + namespace: string; + sessionId: string; + eventId: string; + }): Promise { + const session = this.driver.session(); + try { + await session.run( + `MATCH (s:Session {id: $sid}) + MATCH (e:Event {id: $eid}) + MERGE (s)-[:HAS_EVENT]->(e)`, + { sid: this.sessionNodeId(params.namespace, params.sessionId), eid: params.eventId }, + ); + } finally { + await session.close(); + } + } + + async linkEventTopic(params: { + namespace: string; + eventId: string; + topicName: string; + }): Promise { + const session = this.driver.session(); + try { + await session.run( + `MATCH (e:Event {id: $eid}) + MERGE (t:Topic {id: $tid}) + ON CREATE SET t.name = $name, t.frequency = 0, t.importance = 0, t.namespace = $ns + MERGE (e)-[:ABOUT_TOPIC]->(t)`, + { + eid: params.eventId, + tid: this.topicNodeId(params.namespace, params.topicName), + name: params.topicName, + ns: params.namespace, + }, + ); + } finally { + await session.close(); + } + } + + async linkEventEntity(params: { + namespace: string; + eventId: string; + entityName: string; + entityType: string; + }): Promise { + const session = this.driver.session(); + try { + await session.run( + `MATCH (e:Event {id: $eid}) + MERGE (x:Entity {id: $xid}) + ON CREATE SET x.name = $name, x.type = $type, x.frequency = 0, x.namespace = $ns + MERGE (e)-[:ABOUT_ENTITY]->(x)`, + { + eid: params.eventId, + xid: this.entityNodeId(params.namespace, params.entityType, params.entityName), + name: params.entityName, + type: params.entityType, + ns: params.namespace, + }, + ); + } finally { + await session.close(); + } + } + + async linkSessionTopic(params: { + namespace: string; + sessionId: string; + topicName: string; + }): Promise { + const session = this.driver.session(); + try { + await session.run( + `MATCH (s:Session {id: $sid}) + MERGE (t:Topic {id: $tid}) + ON CREATE SET t.name = $name, t.frequency = 0, t.importance = 0, t.namespace = $ns + MERGE (s)-[:CONTAINS]->(t)`, + { + sid: this.sessionNodeId(params.namespace, params.sessionId), + tid: this.topicNodeId(params.namespace, params.topicName), + name: params.topicName, + ns: params.namespace, + }, + ); + } finally { + await session.close(); + } + } + + async linkTopicEntity(params: { + namespace: string; + topicName: string; + entityName: string; + entityType: string; + }): Promise { + const session = this.driver.session(); + try { + await session.run( + `MATCH (t:Topic {id: $tid}) + MATCH (e:Entity {id: $eid}) + MERGE (t)-[:MENTIONS]->(e)`, + { + tid: this.topicNodeId(params.namespace, params.topicName), + eid: this.entityNodeId(params.namespace, params.entityType, params.entityName), + }, + ); + } finally { + await session.close(); + } + } + + async upsertMemory(params: { + namespace: string; + id: string; + memory: CandidateMemory; + sessionId: string; + }): Promise { + const session = this.driver.session(); + try { + await session.run( + `MATCH (s:Session {id: $sid}) + MERGE (m:Memory {id: $id}) + ON CREATE SET m.content = $content, m.importance = $importance, m.created_at = datetime($created_at), m.frequency = 0, m.namespace = $ns + SET m.content = $content, + m.importance = greatest(coalesce(m.importance, 0), $importance), + m.kind = $kind, + m.memory_key = $memory_key, + m.subject = $subject, + m.confidence = $confidence, + m.expires_at = CASE WHEN $expires_at IS NULL THEN NULL ELSE datetime($expires_at) END, + m.frequency = coalesce(m.frequency, 0) + 1, + m.last_seen_at = datetime() + MERGE (m)-[:FROM_SESSION]->(s)`, + { + sid: this.sessionNodeId(params.namespace, params.sessionId), + id: params.id, + ns: params.namespace, + content: params.memory.content, + importance: params.memory.importance, + created_at: params.memory.createdAt, + kind: params.memory.kind, + memory_key: params.memory.memoryKey ?? null, + subject: params.memory.subject ?? null, + confidence: + typeof params.memory.confidence === "number" ? params.memory.confidence : null, + expires_at: params.memory.expiresAt ?? null, + }, + ); + } finally { + await session.close(); + } + } + + async linkMemoryTopic(params: { + namespace: string; + memoryId: string; + topicName: string; + }): Promise { + const session = this.driver.session(); + try { + await session.run( + `MATCH (m:Memory {id: $mid}) + MERGE (t:Topic {id: $tid}) + ON CREATE SET t.name = $name, t.frequency = 0, t.importance = 0, t.namespace = $ns + MERGE (m)-[:ABOUT_TOPIC]->(t)`, + { + mid: params.memoryId, + tid: this.topicNodeId(params.namespace, params.topicName), + name: params.topicName, + ns: params.namespace, + }, + ); + } finally { + await session.close(); + } + } + + async linkMemoryEntity(params: { + namespace: string; + memoryId: string; + entityName: string; + entityType: string; + }) { + const session = this.driver.session(); + try { + await session.run( + `MATCH (m:Memory {id: $mid}) + MERGE (e:Entity {id: $eid}) + ON CREATE SET e.name = $name, e.type = $type, e.frequency = 0, e.namespace = $ns + MERGE (m)-[:ABOUT_ENTITY]->(e)`, + { + mid: params.memoryId, + eid: this.entityNodeId(params.namespace, params.entityType, params.entityName), + name: params.entityName, + type: params.entityType, + ns: params.namespace, + }, + ); + } finally { + await session.close(); + } + } + + async linkMemoryRelated(params: { + namespace: string; + fromMemoryId: string; + toMemoryId: string; + score: number; + }): Promise { + const session = this.driver.session(); + try { + await session.run( + `MATCH (a:Memory {id: $a}) + MATCH (b:Memory {id: $b}) + MERGE (a)-[r:RELATED_TO]->(b) + SET r.score = greatest(coalesce(r.score, 0), $score), + r.updated_at = datetime()`, + { a: params.fromMemoryId, b: params.toMemoryId, score: params.score }, + ); + } finally { + await session.close(); + } + } + + /** + * Relation query: given entities/topics, return related memories and a relationScore. + * Score is heuristic 0..1 based on matched signals. + */ + async queryRelatedMemories(params: { + namespace: string; + entities: string[]; + topics: string[]; + limit: number; + }): Promise< + Array<{ + id: string; + content: string; + importance: number; + frequency: number; + lastSeenAt: string; + relationScore: number; + kind?: string; + memoryKey?: string; + subject?: string; + expiresAt?: string; + confidence?: number; + }> + > { + const session = this.driver.session(); + try { + const res = await session.run( + `WITH $entities AS entities, $topics AS topics, $prefix AS prefix + // Direct links: Memory -> Entity/Topic + CALL { + WITH entities, topics, prefix + MATCH (m:Memory)-[:ABOUT_ENTITY]->(e:Entity) + WHERE e.name IN entities AND m.id STARTS WITH prefix + RETURN m, 1.0 AS score + UNION ALL + WITH entities, topics, prefix + MATCH (m:Memory)-[:ABOUT_TOPIC]->(t:Topic) + WHERE t.name IN topics AND m.id STARTS WITH prefix + RETURN m, 0.8 AS score + } + WITH m, sum(score) AS directScore + // Two-hop: Memory->Topic->Entity or Memory->Entity<-MENTIONS-Topic + CALL { + WITH m, entities, topics + OPTIONAL MATCH (m:Memory)-[:ABOUT_TOPIC]->(t:Topic)-[:MENTIONS]->(e:Entity) + WHERE e.name IN entities + RETURN coalesce(count(e), 0) AS hopHits + } + WITH m, directScore, hopHits + // RELATED_TO expansion (synapse links) + CALL { + WITH m + OPTIONAL MATCH (m)-[r:RELATED_TO]->(m2:Memory) + RETURN coalesce(max(r.score), 0.0) AS relatedBoost + } + WITH m, + (directScore + toFloat(hopHits) * 0.4 + relatedBoost * 0.6) AS rawScore + WHERE rawScore > 0 + RETURN m.id AS id, + m.content AS content, + coalesce(m.importance, 0.0) AS importance, + coalesce(m.frequency, 0) AS frequency, + toString(coalesce(m.last_seen_at, m.created_at)) AS lastSeenAt, + toString(coalesce(m.expires_at, "")) AS expiresAt, + coalesce(m.kind, "") AS kind, + coalesce(m.memory_key, "") AS memoryKey, + coalesce(m.subject, "") AS subject, + coalesce(m.confidence, 0.0) AS confidence, + least(1.0, rawScore / 2.0) AS relationScore + ORDER BY relationScore DESC, importance DESC + LIMIT $limit`, + { + entities: params.entities, + topics: params.topics, + prefix: this.prefix(params.namespace), + limit: params.limit, + }, + ); + return res.records.map((r) => ({ + id: String(r.get("id")), + content: String(r.get("content")), + importance: Number(r.get("importance") ?? 0), + frequency: Number(r.get("frequency") ?? 0), + lastSeenAt: String(r.get("lastSeenAt") ?? ""), + expiresAt: String(r.get("expiresAt") ?? "") || undefined, + kind: String(r.get("kind") ?? "") || undefined, + memoryKey: String(r.get("memoryKey") ?? "") || undefined, + subject: String(r.get("subject") ?? "") || undefined, + confidence: Number(r.get("confidence") ?? 0) || undefined, + relationScore: Number(r.get("relationScore") ?? 0), + })); + } finally { + await session.close(); + } + } + + async deleteMemoriesByIds(params: { namespace: string; ids: string[] }): Promise { + const ids = params.ids.filter((id) => id.startsWith(this.prefix(params.namespace))); + if (ids.length === 0) { + return 0; + } + const session = this.driver.session(); + try { + const res = await session.run( + `MATCH (m:Memory) + WHERE m.id IN $ids + DETACH DELETE m + RETURN count(*) AS deleted`, + { ids }, + ); + const row = res.records[0]; + const deleted = row ? Number(row.get("deleted") ?? 0) : 0; + return Number.isFinite(deleted) ? deleted : 0; + } finally { + await session.close(); + } + } + + async deleteMemoriesBySession(params: { namespace: string; sessionId: string }): Promise { + const session = this.driver.session(); + try { + const res = await session.run( + `MATCH (m:Memory)-[:FROM_SESSION]->(s:Session {id: $sid}) + DETACH DELETE m + RETURN count(*) AS deleted`, + { sid: this.sessionNodeId(params.namespace, params.sessionId) }, + ); + const row = res.records[0]; + const deleted = row ? Number(row.get("deleted") ?? 0) : 0; + return Number.isFinite(deleted) ? deleted : 0; + } finally { + await session.close(); + } + } + + async scanMemories(params: { namespace?: string; afterId?: string; limit: number }): Promise< + Array<{ + id: string; + namespace: string; + content: string; + kind?: string; + memoryKey?: string; + subject?: string; + expiresAt?: string; + confidence?: number; + importance: number; + frequency: number; + createdAt: string; + sessionId?: string; + topics: string[]; + entities: Array<{ name: string; type?: string }>; + }> + > { + const limit = Math.max(1, Math.min(1000, Math.floor(params.limit))); + const session = this.driver.session(); + try { + const res = await session.run( + ` + MATCH (m:Memory) + WHERE ($ns IS NULL OR m.namespace = $ns) + AND ($afterId IS NULL OR m.id > $afterId) + OPTIONAL MATCH (m)-[:FROM_SESSION]->(s:Session) + OPTIONAL MATCH (m)-[:ABOUT_TOPIC]->(t:Topic) + OPTIONAL MATCH (m)-[:ABOUT_ENTITY]->(e:Entity) + WITH m, s, + collect(DISTINCT t.name) AS topics, + collect(DISTINCT {name: e.name, type: e.type}) AS entities + RETURN + m.id AS id, + coalesce(m.namespace, "") AS namespace, + coalesce(m.content, "") AS content, + toString(coalesce(m.created_at, datetime())) AS createdAt, + coalesce(m.kind, "") AS kind, + coalesce(m.memory_key, "") AS memoryKey, + coalesce(m.subject, "") AS subject, + toString(coalesce(m.expires_at, "")) AS expiresAt, + coalesce(m.confidence, 0.0) AS confidence, + coalesce(m.importance, 0.0) AS importance, + coalesce(m.frequency, 0) AS frequency, + coalesce(s.id, "") AS sessionNodeId, + topics AS topics, + entities AS entities + ORDER BY id ASC + LIMIT $limit + `, + { + ns: params.namespace ?? null, + afterId: params.afterId ?? null, + limit, + }, + ); + + return res.records.map((r) => { + const sessionNodeId = String(r.get("sessionNodeId") ?? ""); + const topicsRaw = r.get("topics") as unknown; + const entitiesRaw = r.get("entities") as unknown; + + const topics = Array.isArray(topicsRaw) + ? topicsRaw.map((t) => String(t)).filter((t) => t.trim().length > 0) + : []; + const entities = Array.isArray(entitiesRaw) + ? entitiesRaw + .map((x) => { + if (typeof x !== "object" || !x) { + return null; + } + const rec = x as Record; + const name = typeof rec.name === "string" ? rec.name.trim() : ""; + if (!name) { + return null; + } + const type = typeof rec.type === "string" ? rec.type.trim() : ""; + return type ? { name, type } : { name }; + }) + .filter((e): e is { name: string; type?: string } => e !== null) + : []; + + const kindRaw = String(r.get("kind") ?? ""); + const memoryKeyRaw = String(r.get("memoryKey") ?? ""); + const subjectRaw = String(r.get("subject") ?? ""); + const expiresAtRaw = String(r.get("expiresAt") ?? ""); + + const confidenceRaw = r.get("confidence") as unknown; + const confidence = + typeof confidenceRaw === "number" && Number.isFinite(confidenceRaw) + ? confidenceRaw + : undefined; + + return { + id: String(r.get("id") ?? ""), + namespace: String(r.get("namespace") ?? ""), + content: String(r.get("content") ?? ""), + createdAt: String(r.get("createdAt") ?? new Date().toISOString()), + kind: kindRaw || undefined, + memoryKey: memoryKeyRaw || undefined, + subject: subjectRaw || undefined, + expiresAt: expiresAtRaw || undefined, + confidence, + importance: Number(r.get("importance") ?? 0), + frequency: Number(r.get("frequency") ?? 0), + sessionId: sessionNodeId ? this.parseSessionIdFromNodeId(sessionNodeId) : undefined, + topics, + entities, + }; + }); + } finally { + await session.close(); + } + } + + async countMemories(params: { namespace?: string }): Promise { + const session = this.driver.session(); + try { + const res = await session.run( + ` + MATCH (m:Memory) + WHERE ($ns IS NULL OR m.namespace = $ns) + RETURN count(m) AS cnt + `, + { ns: params.namespace ?? null }, + ); + const row = res.records[0]; + const cnt = row ? Number(row.get("cnt") ?? 0) : 0; + return Number.isFinite(cnt) ? cnt : 0; + } finally { + await session.close(); + } + } + + async listNamespaces(params?: { limit?: number }): Promise { + const limit = Math.max(1, Math.min(10_000, Math.floor(params?.limit ?? 1000))); + const session = this.driver.session(); + try { + const res = await session.run( + ` + MATCH (m:Memory) + WITH DISTINCT coalesce(m.namespace, "") AS ns + WHERE ns <> "" + RETURN ns + ORDER BY ns ASC + LIMIT $limit + `, + { limit }, + ); + return res.records.map((r) => String(r.get("ns") ?? "")).filter((s) => s.trim().length > 0); + } finally { + await session.close(); + } + } +} diff --git a/packages/deep-memory-server/src/qdrant.ts b/packages/deep-memory-server/src/qdrant.ts new file mode 100644 index 0000000000000..e01302945c622 --- /dev/null +++ b/packages/deep-memory-server/src/qdrant.ts @@ -0,0 +1,286 @@ +import { QdrantClient } from "@qdrant/js-client-rest"; +import type { SchemaCheckResult } from "./schema.js"; + +export type QdrantMemoryPayload = { + id: string; + namespace: string; + kind?: string; + memory_key?: string; + subject?: string; + expires_at?: string; + confidence?: number; + content: string; + session_id: string; + source_transcript_hash?: string; + source_message_count?: number; + created_at: string; + updated_at?: string; + importance: number; + frequency?: number; + entities: string[]; + topics: string[]; +}; + +export class QdrantStore { + private readonly client: QdrantClient; + private readonly collection: string; + private readonly dims: number; + + constructor(params: { url: string; apiKey?: string; collection: string; dims: number }) { + this.client = new QdrantClient({ + url: params.url, + apiKey: params.apiKey, + }); + this.collection = params.collection; + this.dims = params.dims; + } + + async schemaStatus(params: { + mode: SchemaCheckResult["mode"]; + expectedVersion: number; + }): Promise { + const actions: string[] = []; + const warnings: string[] = []; + try { + const existing = await this.client.getCollections(); + const has = existing.collections.some((c) => c.name === this.collection); + if (!has) { + if (params.mode === "apply") { + await this.client.createCollection(this.collection, { + vectors: { size: this.dims, distance: "Cosine" }, + }); + actions.push(`created collection ${this.collection}`); + return { + ok: true, + mode: params.mode, + expectedVersion: params.expectedVersion, + actions, + warnings, + }; + } + warnings.push(`missing collection ${this.collection}`); + return { + ok: false, + mode: params.mode, + expectedVersion: params.expectedVersion, + actions, + warnings, + }; + } + + // Validate collection vector size matches configured dims. + const info = await this.client.getCollection(this.collection); + const vectors = (info as unknown as { config?: { params?: { vectors?: unknown } } }).config + ?.params?.vectors; + const size = + typeof vectors === "object" && vectors && "size" in vectors + ? (vectors as Record).size + : undefined; + const actualDims = typeof size === "number" ? size : undefined; + if (actualDims != null && actualDims !== this.dims) { + warnings.push( + `collection dims mismatch: expected=${this.dims} actual=${actualDims} (${this.collection})`, + ); + warnings.push( + "migration required: create a new collection with the new dims and reindex memories", + ); + return { + ok: false, + mode: params.mode, + expectedVersion: params.expectedVersion, + actions, + warnings, + }; + } + + return { + ok: true, + mode: params.mode, + expectedVersion: params.expectedVersion, + actions, + warnings, + }; + } catch (err) { + return { + ok: false, + mode: params.mode, + expectedVersion: params.expectedVersion, + actions, + warnings, + error: err instanceof Error ? err.message : String(err), + }; + } + } + + async ensureCollection(): Promise { + // Back-compat: keep behavior (create if missing). + await this.schemaStatus({ mode: "apply", expectedVersion: 0 }); + } + + async healthCheck(): Promise<{ ok: true } | { ok: false; error: string }> { + try { + await this.client.getCollections(); + return { ok: true }; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : String(err) }; + } + } + + async upsertMemory(params: { + id: string; + vector: number[]; + payload: QdrantMemoryPayload; + }): Promise { + await this.client.upsert(this.collection, { + wait: true, + points: [ + { + id: params.id, + vector: params.vector, + payload: params.payload, + }, + ], + }); + } + + async getMemory(id: string): Promise<{ id: string; payload?: QdrantMemoryPayload } | null> { + const res = await this.client.retrieve(this.collection, { + ids: [id], + with_payload: true, + with_vector: false, + }); + const hit = res[0]; + if (!hit) { + return null; + } + return { + id: String(hit.id), + payload: (hit.payload as QdrantMemoryPayload | undefined) ?? undefined, + }; + } + + async search(params: { + vector: number[]; + limit: number; + minScore: number; + namespace?: string; + }): Promise> { + const res = await this.client.search(this.collection, { + vector: params.vector, + limit: params.limit, + with_payload: true, + score_threshold: params.minScore, + ...(params.namespace + ? { + filter: { + must: [ + { + key: "namespace", + match: { value: params.namespace }, + }, + ], + }, + } + : {}), + }); + return res.map((r) => ({ + id: String(r.id), + score: r.score ?? 0, + payload: (r.payload as QdrantMemoryPayload | undefined) ?? undefined, + })); + } + + async deleteByIds(params: { ids: string[] }): Promise { + const ids = params.ids.filter(Boolean); + if (ids.length === 0) { + return 0; + } + await this.client.delete(this.collection, { + wait: true, + points: ids, + }); + return ids.length; + } + + async deleteBySession(params: { namespace: string; sessionId: string }): Promise { + await this.client.delete(this.collection, { + wait: true, + filter: { + must: [ + { key: "namespace", match: { value: params.namespace } }, + { key: "session_id", match: { value: params.sessionId } }, + ], + }, + }); + } + + async listMemoriesBySession(params: { + namespace: string; + sessionId: string; + limit: number; + }): Promise> { + const limit = Math.max(1, Math.min(1000, Math.trunc(params.limit))); + const points: Array<{ id: string; payload?: QdrantMemoryPayload }> = []; + let offset: unknown = undefined; + while (points.length < limit) { + const batch = Math.min(128, limit - points.length); + const res = await ( + this.client as unknown as { + scroll: ( + collection: string, + args: Record, + ) => Promise<{ points?: unknown[]; next_page_offset?: unknown }>; + } + ).scroll(this.collection, { + limit: batch, + with_payload: true, + with_vector: false, + offset, + filter: { + must: [ + { key: "namespace", match: { value: params.namespace } }, + { key: "session_id", match: { value: params.sessionId } }, + ], + }, + }); + const batchPoints = Array.isArray(res?.points) ? res.points : []; + for (const p of batchPoints) { + const po = p as { id?: unknown; payload?: unknown }; + if (!po?.id) { + continue; + } + if (typeof po.id !== "string" && typeof po.id !== "number") { + continue; + } + points.push({ + id: typeof po.id === "number" ? String(po.id) : po.id, + payload: (po.payload as QdrantMemoryPayload | undefined) ?? undefined, + }); + if (points.length >= limit) { + break; + } + } + if (!res?.next_page_offset) { + break; + } + offset = res.next_page_offset; + } + return points; + } + + async countMemories(params?: { namespace?: string }): Promise { + const res = await this.client.count(this.collection, { + exact: true, + ...(params?.namespace + ? { + filter: { + must: [{ key: "namespace", match: { value: params.namespace } }], + }, + } + : {}), + }); + const cnt = (res as unknown as { count?: unknown }).count; + const n = typeof cnt === "number" ? cnt : Number(cnt ?? 0); + return Number.isFinite(n) ? n : 0; + } +} diff --git a/packages/deep-memory-server/src/rate-limit.ts b/packages/deep-memory-server/src/rate-limit.ts new file mode 100644 index 0000000000000..4fdc1d324f4b0 --- /dev/null +++ b/packages/deep-memory-server/src/rate-limit.ts @@ -0,0 +1,41 @@ +type Bucket = { + windowStartMs: number; + count: number; +}; + +export class FixedWindowRateLimiter { + private readonly windowMs: number; + private readonly buckets = new Map(); + + constructor(params: { windowMs: number }) { + this.windowMs = Math.max(1000, Math.floor(params.windowMs)); + } + + take(params: { + key: string; + limit: number; + nowMs?: number; + }): { ok: true; remaining: number; resetAtMs: number } | { ok: false; resetAtMs: number } { + const limit = Math.max(0, Math.floor(params.limit)); + const now = params.nowMs ?? Date.now(); + const windowStart = Math.floor(now / this.windowMs) * this.windowMs; + const resetAt = windowStart + this.windowMs; + if (limit === 0) { + return { ok: true, remaining: 0, resetAtMs: resetAt }; + } + + const existing = this.buckets.get(params.key); + const b = + existing && existing.windowStartMs === windowStart + ? existing + : { windowStartMs: windowStart, count: 0 }; + + if (b.count >= limit) { + this.buckets.set(params.key, b); + return { ok: false, resetAtMs: resetAt }; + } + b.count += 1; + this.buckets.set(params.key, b); + return { ok: true, remaining: Math.max(0, limit - b.count), resetAtMs: resetAt }; + } +} diff --git a/packages/deep-memory-server/src/reindex.ts b/packages/deep-memory-server/src/reindex.ts new file mode 100644 index 0000000000000..13101b57f9c5c --- /dev/null +++ b/packages/deep-memory-server/src/reindex.ts @@ -0,0 +1,418 @@ +import fs from "node:fs/promises"; +import process from "node:process"; +import { loadConfig } from "./config.js"; +import { EmbeddingModel } from "./embeddings.js"; +import { createLogger } from "./logger.js"; +import { Neo4jStore } from "./neo4j.js"; +import { QdrantStore, type QdrantMemoryPayload } from "./qdrant.js"; +import { DEEPMEM_SCHEMA_VERSION } from "./schema.js"; + +type Args = { + namespace?: string; + allNamespaces: boolean; + targetCollection?: string; + batchSize: number; + maxItems?: number; + dryRun: boolean; + concurrency: number; + maxRetries: number; + continueOnError: boolean; + cursorFile?: string; + resume: boolean; + reportFile?: string; +}; + +function parseArgs(argv: string[]): Args { + const out: Args = { + allNamespaces: false, + batchSize: 100, + dryRun: false, + concurrency: 4, + maxRetries: 3, + continueOnError: false, + resume: false, + }; + + for (let i = 0; i < argv.length; i += 1) { + const t = argv[i]?.trim(); + if (!t) { + continue; + } + if (t === "--namespace" || t === "-n") { + out.namespace = argv[i + 1]?.trim(); + i += 1; + continue; + } + if (t === "--all-namespaces" || t === "--all") { + out.allNamespaces = true; + continue; + } + if (t === "--target-collection" || t === "--collection") { + out.targetCollection = argv[i + 1]?.trim(); + i += 1; + continue; + } + if (t === "--batch-size") { + const v = Number(argv[i + 1]); + if (Number.isFinite(v) && v > 0) { + out.batchSize = Math.max(1, Math.min(1000, Math.floor(v))); + } + i += 1; + continue; + } + if (t === "--concurrency" || t === "--parallel") { + const v = Number(argv[i + 1]); + if (Number.isFinite(v) && v > 0) { + out.concurrency = Math.max(1, Math.min(32, Math.floor(v))); + } + i += 1; + continue; + } + if (t === "--max-items") { + const v = Number(argv[i + 1]); + if (Number.isFinite(v) && v > 0) { + out.maxItems = Math.max(1, Math.floor(v)); + } + i += 1; + continue; + } + if (t === "--cursor-file") { + const v = argv[i + 1]?.trim(); + if (v) { + out.cursorFile = v; + } + i += 1; + continue; + } + if (t === "--resume") { + out.resume = true; + continue; + } + if (t === "--max-retries") { + const v = Number(argv[i + 1]); + if (Number.isFinite(v) && v >= 0) { + out.maxRetries = Math.max(0, Math.min(10, Math.floor(v))); + } + i += 1; + continue; + } + if (t === "--continue-on-error") { + out.continueOnError = true; + continue; + } + if (t === "--report-file") { + const v = argv[i + 1]?.trim(); + if (v) { + out.reportFile = v; + } + i += 1; + continue; + } + if (t === "--dry-run") { + out.dryRun = true; + continue; + } + if (t === "--help" || t === "-h") { + printHelpAndExit(0); + } + } + + if (!out.allNamespaces) { + out.namespace = out.namespace?.trim() || undefined; + if (!out.namespace) { + throw new Error("missing --namespace (or use --all-namespaces)"); + } + } + return out; +} + +function printHelpAndExit(code: number): never { + // eslint-disable-next-line no-console + console.log(` +Usage: pnpm --dir packages/deep-memory-server reindex -- --namespace [options] + +Options: + --namespace, -n Reindex a single namespace (recommended) + --all-namespaces, --all Reindex all namespaces (large; use with care) + --target-collection Qdrant collection to write into (default: QDRANT_COLLECTION) + --batch-size Neo4j page size (default: 100, max: 1000) + --concurrency, --parallel Embed+upsert concurrency (default: 4, max: 32) + --max-items Stop after N items (for canary) + --cursor-file Persist cursor JSON for resume (optional) + --resume Resume from cursor-file if it exists + --max-retries Retries per item on embed/upsert failure (default: 3) + --continue-on-error Continue past failures and write a report (default: stop) + --report-file Write a JSON summary report (optional) + --dry-run Scan only; do not embed or write + --help, -h Show help +`); + process.exit(code); +} + +type CursorState = { + afterId?: string; + processed: number; + written: number; + failed: number; + updatedAt: string; +}; + +async function sleep(ms: number) { + return await new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function writeCursor(file: string, state: CursorState) { + const body = `${JSON.stringify(state, null, 2)}\n`; + await fs.writeFile(file, body, "utf8"); +} + +async function readCursor(file: string): Promise { + try { + const raw = await fs.readFile(file, "utf8"); + const parsed = JSON.parse(raw) as Partial; + if (typeof parsed !== "object" || !parsed) { + return null; + } + return { + afterId: typeof parsed.afterId === "string" ? parsed.afterId : undefined, + processed: Number(parsed.processed ?? 0) || 0, + written: Number(parsed.written ?? 0) || 0, + failed: Number(parsed.failed ?? 0) || 0, + updatedAt: typeof parsed.updatedAt === "string" ? parsed.updatedAt : new Date().toISOString(), + }; + } catch { + return null; + } +} + +async function withRetries(params: { maxRetries: number; fn: () => Promise }): Promise { + let attempt = 0; + // maxRetries=0 means "try once" + while (true) { + attempt += 1; + try { + return await params.fn(); + } catch (err) { + if (attempt > Math.max(1, params.maxRetries + 1)) { + throw err; + } + const delay = Math.min(10_000, 250 * 2 ** Math.min(10, attempt - 1)); + await sleep(delay); + } + } +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const cfg = loadConfig(); + const log = createLogger(cfg.LOG_LEVEL); + + const targetCollection = (args.targetCollection ?? cfg.QDRANT_COLLECTION).trim(); + if (!targetCollection) { + throw new Error("empty target collection"); + } + + const neo4j = new Neo4jStore({ + uri: cfg.NEO4J_URI, + user: cfg.NEO4J_USER, + password: cfg.NEO4J_PASSWORD, + }); + + const qdrant = new QdrantStore({ + url: cfg.QDRANT_URL, + apiKey: cfg.QDRANT_API_KEY, + collection: targetCollection, + dims: cfg.VECTOR_DIMS, + }); + + const embedder = new EmbeddingModel({ modelId: cfg.EMBEDDING_MODEL, dims: cfg.VECTOR_DIMS }); + + log.info( + { + namespace: args.allNamespaces ? "*" : args.namespace, + batchSize: args.batchSize, + maxItems: args.maxItems, + dryRun: args.dryRun, + concurrency: args.concurrency, + maxRetries: args.maxRetries, + continueOnError: args.continueOnError, + cursorFile: args.cursorFile, + resume: args.resume, + reportFile: args.reportFile, + targetCollection, + dims: cfg.VECTOR_DIMS, + model: cfg.EMBEDDING_MODEL, + }, + "deep-memory reindex starting", + ); + + const qdrantSchema = await qdrant.schemaStatus({ + mode: "apply", + expectedVersion: DEEPMEM_SCHEMA_VERSION, + }); + if (!qdrantSchema.ok) { + throw new Error(`qdrant schema not ready: ${qdrantSchema.error ?? "unknown"}`); + } + + const cursorFile = args.cursorFile?.trim() || undefined; + let afterId: string | undefined; + let processed = 0; + let written = 0; + let failed = 0; + if (cursorFile) { + const existing = await readCursor(cursorFile); + if (existing && args.resume) { + afterId = existing.afterId; + processed = existing.processed; + written = existing.written; + failed = existing.failed; + log.info({ cursorFile, afterId, processed, written, failed }, "reindex resumed from cursor"); + } else if (existing && !args.resume) { + throw new Error(`cursor file exists; pass --resume to continue (${cursorFile})`); + } + } + + const failures: Array<{ id: string; namespace: string; error: string }> = []; + + try { + while (true) { + const page = await neo4j.scanMemories({ + namespace: args.allNamespaces ? undefined : args.namespace, + afterId, + limit: args.batchSize, + }); + if (page.length === 0) { + break; + } + + const tasks = page.map((m) => async () => { + if (args.dryRun) { + return; + } + const vec = await withRetries({ + maxRetries: args.maxRetries, + fn: async () => await embedder.embed(m.content), + }); + const payload: QdrantMemoryPayload = { + id: m.id, + namespace: m.namespace, + kind: m.kind, + memory_key: m.memoryKey, + subject: m.subject, + expires_at: m.expiresAt, + confidence: m.confidence, + content: m.content, + session_id: m.sessionId ?? "", + created_at: m.createdAt, + updated_at: new Date().toISOString(), + importance: m.importance, + frequency: m.frequency, + entities: m.entities.map((e) => e.name).slice(0, 10), + topics: m.topics.slice(0, 10), + }; + await withRetries({ + maxRetries: args.maxRetries, + fn: async () => await qdrant.upsertMemory({ id: m.id, vector: vec, payload }), + }); + }); + + const results: Array< + Promise<{ ok: boolean; id: string; namespace: string; error?: string }> + > = []; + for (const m of page) { + const fn = tasks[results.length]; + if (!fn) { + break; + } + results.push( + fn() + .then(() => ({ ok: true as const, id: m.id, namespace: m.namespace })) + .catch((err) => ({ + ok: false as const, + id: m.id, + namespace: m.namespace, + error: err instanceof Error ? (err.stack ?? err.message) : String(err), + })), + ); + } + + // Basic concurrency throttle while preserving cursor advancement order. + const inflight = new Set>(); + let launched = 0; + const launchUntil = () => { + while (launched < results.length && inflight.size < args.concurrency) { + const p = results[launched]; + if (!p) { + launched += 1; + continue; + } + inflight.add(p); + void p.finally(() => inflight.delete(p)); + launched += 1; + } + }; + launchUntil(); + + for (let i = 0; i < results.length; i += 1) { + const r = await results[i]; + if (!r) { + continue; + } + processed += 1; + afterId = r.id; + if (!args.dryRun && r.ok) { + written += 1; + } + if (!r.ok) { + failed += 1; + failures.push({ id: r.id, namespace: r.namespace, error: r.error ?? "unknown" }); + log.error({ id: r.id, err: r.error }, "reindex item failed"); + if (!args.continueOnError) { + throw new Error(`reindex failed at id=${r.id}`); + } + } + if (cursorFile && (processed % 20 === 0 || i === results.length - 1)) { + await writeCursor(cursorFile, { + afterId, + processed, + written, + failed, + updatedAt: new Date().toISOString(), + }); + } + if (args.maxItems && processed >= args.maxItems) { + break; + } + launchUntil(); + } + + if (args.maxItems && processed > args.maxItems) { + break; + } + } + } finally { + await neo4j.close().catch(() => {}); + } + + const report = { + ok: failed === 0, + namespace: args.allNamespaces ? "*" : args.namespace, + targetCollection, + processed, + written, + failed, + afterId, + failures: args.continueOnError ? failures : undefined, + finishedAt: new Date().toISOString(), + }; + if (args.reportFile?.trim()) { + await fs.writeFile(args.reportFile.trim(), `${JSON.stringify(report, null, 2)}\n`, "utf8"); + } + log.info(report, "deep-memory reindex finished"); +} + +void main().catch((err) => { + // eslint-disable-next-line no-console + console.error(String(err instanceof Error ? (err.stack ?? err.message) : err)); + process.exit(1); +}); diff --git a/packages/deep-memory-server/src/retriever.conflict.test.ts b/packages/deep-memory-server/src/retriever.conflict.test.ts new file mode 100644 index 0000000000000..d3e2bf0f2eda5 --- /dev/null +++ b/packages/deep-memory-server/src/retriever.conflict.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from "vitest"; +import type { EmbeddingModel } from "./embeddings.js"; +import type { Neo4jStore } from "./neo4j.js"; +import type { QdrantStore } from "./qdrant.js"; +import { DeepMemoryRetriever } from "./retriever.js"; + +describe("DeepMemoryRetriever conflict resolution", () => { + it("dedupes by memory_key and drops expired", async () => { + const embedder = { embed: async () => [0, 0, 0] }; + const qdrant = { + search: async () => [ + { + id: "a", + score: 0.9, + payload: { + id: "a", + namespace: "default", + kind: "preference", + memory_key: "preference:timezone", + subject: "timezone", + content: "User prefers timezone UTC+8", + session_id: "s0", + created_at: "2020-01-01T00:00:00.000Z", + updated_at: "2020-01-01T00:00:00.000Z", + importance: 0.9, + frequency: 1, + entities: [], + topics: [], + }, + }, + { + id: "b", + score: 0.88, + payload: { + id: "b", + namespace: "default", + kind: "preference", + memory_key: "preference:timezone", + subject: "timezone", + content: "User prefers timezone UTC", + session_id: "s0", + created_at: "2021-01-01T00:00:00.000Z", + updated_at: "2021-01-01T00:00:00.000Z", + importance: 0.8, + frequency: 1, + entities: [], + topics: [], + }, + }, + { + id: "c", + score: 0.95, + payload: { + id: "c", + namespace: "default", + kind: "ephemeral", + memory_key: "ephemeral:test", + subject: "test", + expires_at: "2000-01-01T00:00:00.000Z", + content: "Temporary info (expired)", + session_id: "s0", + created_at: "1999-01-01T00:00:00.000Z", + updated_at: "1999-01-01T00:00:00.000Z", + importance: 1, + frequency: 1, + entities: [], + topics: [], + }, + }, + ], + }; + const neo4j = { queryRelatedMemories: async () => [] }; + + const retriever = new DeepMemoryRetriever({ + embedder: embedder as unknown as EmbeddingModel, + qdrant: qdrant as unknown as QdrantStore, + neo4j: neo4j as unknown as Neo4jStore, + minSemanticScore: 0, + semanticWeight: 1, + relationWeight: 0, + decayHalfLifeDays: 90, + importanceBoost: 0, + frequencyBoost: 0, + }); + + const out = await retriever.retrieve({ + namespace: "default", + userInput: "timezone", + sessionId: "s1", + maxMemories: 10, + entities: [], + topics: [], + }); + + // one of a/b survives due to same memory_key; expired c is dropped. + expect(out.memories.some((m) => m.id === "c")).toBe(false); + const tz = out.memories.filter((m) => m.id === "a" || m.id === "b"); + expect(tz.length).toBe(1); + }); +}); diff --git a/packages/deep-memory-server/src/retriever.test.ts b/packages/deep-memory-server/src/retriever.test.ts new file mode 100644 index 0000000000000..1fdefd0383785 --- /dev/null +++ b/packages/deep-memory-server/src/retriever.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; +import type { EmbeddingModel } from "./embeddings.js"; +import type { Neo4jStore } from "./neo4j.js"; +import type { QdrantStore } from "./qdrant.js"; +import { DeepMemoryRetriever } from "./retriever.js"; + +describe("DeepMemoryRetriever", () => { + it("merges semantic + relation and applies decay/boosts", async () => { + const embedder = { + embed: async () => [0, 0, 0], + }; + const qdrant = { + search: async () => [ + { + id: "m1", + score: 0.9, + payload: { + id: "m1", + namespace: "default", + kind: "fact", + memory_key: "fact:topic", + subject: "topic", + content: "semantic memory", + session_id: "s0", + created_at: "2020-01-01T00:00:00.000Z", + updated_at: "2020-01-01T00:00:00.000Z", + importance: 0.9, + frequency: 9, + entities: [], + topics: [], + }, + }, + ], + }; + const neo4j = { + queryRelatedMemories: async () => [ + { + id: "m2", + content: "relation memory", + importance: 0.9, + frequency: 9, + lastSeenAt: new Date().toISOString(), + kind: "rule", + memoryKey: "rule:topic", + subject: "topic", + relationScore: 1.0, + }, + ], + }; + + const retriever = new DeepMemoryRetriever({ + embedder: embedder as unknown as EmbeddingModel, + qdrant: qdrant as unknown as QdrantStore, + neo4j: neo4j as unknown as Neo4jStore, + minSemanticScore: 0, + semanticWeight: 0.6, + relationWeight: 0.4, + decayHalfLifeDays: 90, + importanceBoost: 0.3, + frequencyBoost: 0.2, + }); + + const out = await retriever.retrieve({ + namespace: "default", + userInput: "x", + sessionId: "s1", + maxMemories: 2, + entities: ["x"], + topics: ["x"], + }); + + expect(out.memories.length).toBe(2); + // m1 has high semantic but is very old => decayed heavily. + // m2 has strong relation and is fresh => should rank above m1. + expect(out.memories[0]?.id).toBe("m2"); + expect(out.memories[1]?.id).toBe("m1"); + }); +}); diff --git a/packages/deep-memory-server/src/retriever.ts b/packages/deep-memory-server/src/retriever.ts new file mode 100644 index 0000000000000..38d033764d8c8 --- /dev/null +++ b/packages/deep-memory-server/src/retriever.ts @@ -0,0 +1,275 @@ +import { EmbeddingModel } from "./embeddings.js"; +import { Neo4jStore } from "./neo4j.js"; +import { QdrantStore } from "./qdrant.js"; +import type { RetrieveContextResponse } from "./types.js"; +import { clamp } from "./utils.js"; + +export class DeepMemoryRetriever { + private readonly embedder: EmbeddingModel; + private readonly qdrant: QdrantStore; + private readonly neo4j: Neo4jStore; + private readonly minSemanticScore: number; + private readonly semanticWeight: number; + private readonly relationWeight: number; + private readonly decayHalfLifeDays: number; + private readonly importanceBoost: number; + private readonly frequencyBoost: number; + + constructor(params: { + embedder: EmbeddingModel; + qdrant: QdrantStore; + neo4j: Neo4jStore; + minSemanticScore: number; + semanticWeight: number; + relationWeight: number; + decayHalfLifeDays: number; + importanceBoost: number; + frequencyBoost: number; + }) { + this.embedder = params.embedder; + this.qdrant = params.qdrant; + this.neo4j = params.neo4j; + this.minSemanticScore = params.minSemanticScore; + const sum = Math.max(0, params.semanticWeight) + Math.max(0, params.relationWeight); + this.semanticWeight = sum > 0 ? params.semanticWeight / sum : 0.6; + this.relationWeight = sum > 0 ? params.relationWeight / sum : 0.4; + this.decayHalfLifeDays = Math.max(1, params.decayHalfLifeDays); + this.importanceBoost = Math.max(0, params.importanceBoost); + this.frequencyBoost = Math.max(0, params.frequencyBoost); + } + + private scoreWithDecay(params: { + relevance: number; + importance: number; + frequency: number; + lastSeenAt?: string; + }): number { + const base = Math.max(0, params.relevance); + const imp = clamp(params.importance ?? 0, 0, 1); + const freq = Math.max(0, params.frequency ?? 0); + const freqNorm = clamp(Math.log1p(freq) / Math.log(10), 0, 1); // ~1 at 9+ + const boost = (1 + this.importanceBoost * imp) * (1 + this.frequencyBoost * freqNorm); + + let decay = 1; + if (params.lastSeenAt) { + const t = Date.parse(params.lastSeenAt); + if (!Number.isNaN(t)) { + const ageDays = Math.max(0, (Date.now() - t) / (24 * 3600_000)); + decay = Math.pow(0.5, ageDays / this.decayHalfLifeDays); + decay = clamp(decay, 0.1, 1); + } + } + return base * boost * decay; + } + + async retrieve(params: { + namespace: string; + userInput: string; + sessionId: string; + maxMemories: number; + entities: string[]; + topics: string[]; + }): Promise { + const semanticWeight = this.semanticWeight; + const relationWeight = this.relationWeight; + + const candidates = Math.min(50, Math.max(10, params.maxMemories * 5)); + type Merged = { + id: string; + content: string; + importance: number; + frequency: number; + lastSeenAt?: string; + kind?: string; + memoryKey?: string; + subject?: string; + expiresAt?: string; + confidence?: number; + semantic: number; + relation: number; + sources: Set<"qdrant" | "neo4j">; + }; + const resultsById = new Map(); + + const getOrInit = ( + id: string, + seed: { + content: string; + importance: number; + frequency?: number; + lastSeenAt?: string; + kind?: string; + memoryKey?: string; + subject?: string; + expiresAt?: string; + confidence?: number; + }, + ): Merged => { + const existing = resultsById.get(id); + if (existing) { + return existing; + } + const created: Merged = { + id, + content: seed.content, + importance: seed.importance, + frequency: seed.frequency ?? 0, + lastSeenAt: seed.lastSeenAt, + kind: seed.kind, + memoryKey: seed.memoryKey, + subject: seed.subject, + expiresAt: seed.expiresAt, + confidence: seed.confidence, + semantic: 0, + relation: 0, + sources: new Set(), + }; + resultsById.set(id, created); + return created; + }; + + // Qdrant semantic retrieval (best-effort). + try { + const vec = await this.embedder.embed(params.userInput); + const hits = await this.qdrant.search({ + vector: vec, + limit: candidates, + minScore: this.minSemanticScore, + namespace: params.namespace, + }); + for (const hit of hits) { + const payload = hit.payload; + if (!payload?.content) { + continue; + } + const existing = getOrInit(hit.id, { + content: payload.content, + importance: payload.importance ?? 0, + frequency: payload.frequency ?? 0, + lastSeenAt: payload.updated_at ?? payload.created_at, + kind: payload.kind, + memoryKey: payload.memory_key, + subject: payload.subject, + expiresAt: payload.expires_at, + confidence: payload.confidence, + }); + existing.content = existing.content || payload.content; + existing.importance = Math.max(existing.importance ?? 0, payload.importance ?? 0); + existing.frequency = Math.max(existing.frequency ?? 0, payload.frequency ?? 0); + existing.lastSeenAt = existing.lastSeenAt ?? payload.updated_at ?? payload.created_at; + existing.semantic = Math.max(existing.semantic ?? 0, hit.score ?? 0); + existing.sources.add("qdrant"); + } + } catch { + // ignore; handled by fallback path + } + + // Neo4j relation retrieval (best-effort). + try { + const related = await this.neo4j.queryRelatedMemories({ + namespace: params.namespace, + entities: params.entities, + topics: params.topics, + limit: candidates, + }); + for (const r of related) { + const existing = getOrInit(r.id, { + content: r.content, + importance: r.importance, + frequency: r.frequency, + lastSeenAt: r.lastSeenAt, + kind: r.kind, + memoryKey: r.memoryKey, + subject: r.subject, + expiresAt: r.expiresAt, + confidence: r.confidence, + }); + existing.content = existing.content || r.content; + existing.importance = Math.max(existing.importance ?? 0, r.importance ?? 0); + existing.frequency = Math.max(existing.frequency ?? 0, r.frequency ?? 0); + existing.lastSeenAt = existing.lastSeenAt ?? r.lastSeenAt; + existing.relation = Math.max(existing.relation ?? 0, r.relationScore ?? 0); + existing.sources.add("neo4j"); + } + } catch { + // ignore + } + + const now = Date.now(); + const isExpired = (expiresAt?: string) => { + if (!expiresAt) { + return false; + } + const t = Date.parse(expiresAt); + return Number.isFinite(t) && t > 0 && t < now; + }; + + const merged = Array.from(resultsById.values()) + .filter((r) => !isExpired(r.expiresAt)) + .map((r) => { + const semantic = r.semantic ?? 0; + const relation = r.relation ?? 0; + const relevance = semanticWeight * semantic + relationWeight * relation; + const final = this.scoreWithDecay({ + relevance, + importance: r.importance, + frequency: r.frequency, + lastSeenAt: r.lastSeenAt, + }); + return { + id: r.id, + content: r.content, + importance: r.importance, + relevance: final, + semantic_score: r.semantic, + relation_score: r.relation, + kind: r.kind, + memory_key: r.memoryKey, + subject: r.subject, + sources: Array.from(r.sources), + }; + }) + .toSorted((a, b) => b.relevance - a.relevance); + + // Conflict resolution: group by memory_key (slot) and pick the best per group. + const byKey = new Map(); + for (const m of merged) { + const k = m.memory_key ?? m.id; + const prev = byKey.get(k); + if (!prev) { + byKey.set(k, m); + continue; + } + // Prefer higher relevance; tie-break by importance. + if (m.relevance > prev.relevance) { + byKey.set(k, m); + } else if (m.relevance === prev.relevance && (m.importance ?? 0) > (prev.importance ?? 0)) { + byKey.set(k, m); + } + } + const resolved = Array.from(byKey.values()) + .toSorted((a, b) => b.relevance - a.relevance) + .slice(0, params.maxMemories); + + const contextLines = resolved.map((m, idx) => { + const score = m.relevance.toFixed(2); + const imp = m.importance.toFixed(2); + return `${idx + 1}. (${score}, imp=${imp}) ${m.content}`; + }); + + return { + entities: params.entities, + topics: params.topics, + memories: resolved.map((m) => ({ + id: m.id, + content: m.content, + importance: m.importance, + relevance: m.relevance, + semantic_score: m.semantic_score, + relation_score: m.relation_score, + sources: m.sources, + })), + context: contextLines.length ? `Relevant long-term memory:\n${contextLines.join("\n")}` : "", + }; + } +} diff --git a/packages/deep-memory-server/src/safety.test.ts b/packages/deep-memory-server/src/safety.test.ts new file mode 100644 index 0000000000000..e9b417239d97b --- /dev/null +++ b/packages/deep-memory-server/src/safety.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { createSensitiveFilter } from "./safety.js"; + +describe("sensitive filter", () => { + it("detects builtin sensitive patterns", () => { + const f = createSensitiveFilter({ + SENSITIVE_RULESET_VERSION: "builtin-v1", + SENSITIVE_ALLOW_REGEX_JSON: undefined, + SENSITIVE_DENY_REGEX_JSON: undefined, + }); + const r = f.detect("api_key=sk_1234567890123456789012345"); + expect(r.sensitive).toBe(true); + expect(r.reasons.length).toBeGreaterThan(0); + }); + + it("allowlist overrides deny patterns", () => { + const f = createSensitiveFilter({ + SENSITIVE_RULESET_VERSION: "test", + SENSITIVE_ALLOW_REGEX_JSON: JSON.stringify(["timezone:\\s*asia/shanghai"]), + SENSITIVE_DENY_REGEX_JSON: JSON.stringify(["timezone:"]), + }); + const r = f.detect("timezone: Asia/Shanghai"); + expect(r.sensitive).toBe(false); + }); +}); diff --git a/packages/deep-memory-server/src/safety.ts b/packages/deep-memory-server/src/safety.ts new file mode 100644 index 0000000000000..02a6157ab24fe --- /dev/null +++ b/packages/deep-memory-server/src/safety.ts @@ -0,0 +1,90 @@ +import type { DeepMemoryServerConfig } from "./config.js"; + +type SensitiveRule = { id: string; re: RegExp }; + +const BUILTIN_DENY: SensitiveRule[] = [ + { id: "private_key_block", re: /-----BEGIN (?:RSA|EC|OPENSSH|PGP|DSA)? ?PRIVATE KEY-----/i }, + { + id: "keyword_assignment", + re: /\b(?:api[_-]?key|secret|password|passwd|token)\b\s*[:=]\s*\S+/i, + }, + { id: "secret_prefix", re: /\b(?:sk|rk)_[A-Za-z0-9]{20,}\b/ }, + { id: "long_hex", re: /\b[A-Fa-f0-9]{32,}\b/ }, + { id: "long_digits", re: /\b\d{14,}\b/ }, + { id: "jwt_like", re: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/ }, +]; + +function compileRegexList(raw: string | undefined, prefix: string): SensitiveRule[] { + if (!raw) { + return []; + } + try { + const parsed = JSON.parse(raw) as unknown; + if (!Array.isArray(parsed)) { + return []; + } + const out: SensitiveRule[] = []; + for (let i = 0; i < parsed.length; i += 1) { + const item = parsed[i]; + if (typeof item !== "string" || !item.trim()) { + continue; + } + try { + out.push({ id: `${prefix}_${i}`, re: new RegExp(item, "i") }); + } catch { + // ignore invalid regex + } + } + return out; + } catch { + return []; + } +} + +export type SensitiveDetection = { + sensitive: boolean; + reasons: string[]; + rulesetVersion: string; +}; + +export type SensitiveFilter = { + version: string; + denyRules: SensitiveRule[]; + allowRules: SensitiveRule[]; + detect: (text: string) => SensitiveDetection; +}; + +export function createSensitiveFilter( + cfg: Pick< + DeepMemoryServerConfig, + "SENSITIVE_RULESET_VERSION" | "SENSITIVE_DENY_REGEX_JSON" | "SENSITIVE_ALLOW_REGEX_JSON" + >, +): SensitiveFilter { + const version = cfg.SENSITIVE_RULESET_VERSION?.trim() || "builtin-v1"; + const deny = [...BUILTIN_DENY, ...compileRegexList(cfg.SENSITIVE_DENY_REGEX_JSON, "deny")]; + const allow = compileRegexList(cfg.SENSITIVE_ALLOW_REGEX_JSON, "allow"); + return { + version, + denyRules: deny, + allowRules: allow, + detect: (text: string): SensitiveDetection => { + const t = text.trim(); + if (!t) { + return { sensitive: false, reasons: [], rulesetVersion: version }; + } + if (allow.some((r) => r.re.test(t))) { + return { sensitive: false, reasons: [], rulesetVersion: version }; + } + const reasons = deny.filter((r) => r.re.test(t)).map((r) => r.id); + return { sensitive: reasons.length > 0, reasons, rulesetVersion: version }; + }, + }; +} + +export function looksSensitive(text: string): boolean { + const t = text.trim(); + if (!t) { + return false; + } + return BUILTIN_DENY.some((p) => p.re.test(t)); +} diff --git a/packages/deep-memory-server/src/schema.ts b/packages/deep-memory-server/src/schema.ts new file mode 100644 index 0000000000000..0ed7097354b1d --- /dev/null +++ b/packages/deep-memory-server/src/schema.ts @@ -0,0 +1,13 @@ +export const DEEPMEM_SCHEMA_VERSION = 1; + +export type MigrationMode = "off" | "validate" | "apply"; + +export type SchemaCheckResult = { + ok: boolean; + mode: MigrationMode; + expectedVersion: number; + currentVersion?: number; + actions?: string[]; + warnings?: string[]; + error?: string; +}; diff --git a/packages/deep-memory-server/src/types.ts b/packages/deep-memory-server/src/types.ts new file mode 100644 index 0000000000000..87221a912847f --- /dev/null +++ b/packages/deep-memory-server/src/types.ts @@ -0,0 +1,204 @@ +export type RetrieveContextRequest = { + namespace?: string; + user_input: string; + session_id: string; + max_memories?: number; +}; + +export type RetrieveContextResponse = { + entities: string[]; + topics: string[]; + memories: Array<{ + id: string; + content: string; + importance: number; + relevance: number; + semantic_score?: number; + relation_score?: number; + sources: Array<"qdrant" | "neo4j">; + }>; + context: string; +}; + +export type UpdateMemoryIndexRequest = { + namespace?: string; + session_id: string; + // OpenClaw sends transcript messages (best-effort). We treat as opaque and extract what we can. + messages: unknown[]; + async?: boolean; + /** + * If true and async=false, include memory ids written/updated in the response. + * This is intended for traceability (e.g. file_id -> memory_ids). + */ + return_memory_ids?: boolean; + /** Maximum memory ids to return (default 200, max 1000). Only applies when return_memory_ids=true. */ + max_memory_ids?: number; +}; + +export type UpdateMemoryIndexResponse = + | { + status: "queued"; + memories_added: 0; + memories_filtered: 0; + degraded?: { + mode: "delayed"; + notBeforeMs: number; + delaySeconds: number; + }; + } + | { + status: "processed"; + memories_added: number; + memories_filtered: number; + memory_ids?: string[]; + memory_ids_truncated?: boolean; + } + | { + status: "skipped"; + memories_added: 0; + memories_filtered: 0; + error: "namespace_write_disabled" | "sampled_out" | "throttled"; + } + | { + status: "error"; + memories_added: 0; + memories_filtered: 0; + error: string; + }; + +export type OverloadResponse = + | { + error: "queue_overloaded"; + pendingApprox: number; + retryAfterSeconds: number; + } + | { + error: "degraded_read_only"; + pendingApprox: number; + retryAfterSeconds: number; + } + | { + error: "namespace_overloaded"; + namespace: string; + active: number; + limit: number; + }; + +export type ForgetRequest = { + namespace?: string; + memory_ids?: string[]; + session_id?: string; + dry_run?: boolean; + async?: boolean; +}; + +export type ForgetResponse = + | { + status: "dry_run"; + namespace: string; + request_id?: string; + delete_ids: number; + delete_session: number; + } + | { + status: "queued"; + namespace: string; + request_id?: string; + key: string; + task_id: string; + delete_ids: number; + delete_session: number; + } + | { + status: "processed"; + namespace: string; + request_id?: string; + deleted: number; + results: unknown; + }; + +export type InspectSessionRequest = { + namespace?: string; + session_id: string; + limit?: number; + include_content?: boolean; +}; + +export type InspectSessionResponse = { + namespace: string; + session_id: string; + totals: { + memories: number; + }; + topics: Array<{ name: string; frequency: number }>; + entities: Array<{ name: string; frequency: number }>; + memories: Array<{ + id: string; + importance?: number; + created_at?: string; + content?: string; + topics?: string[]; + entities?: string[]; + }>; + summary?: string; +}; + +export type ExtractedEntity = { + name: string; + type: "person" | "place" | "organization" | "project" | "concept" | "other"; + frequency: number; +}; + +export type ExtractedTopic = { + name: string; + frequency: number; + importance: number; +}; + +export type ExtractedEvent = { + type: + | "requirement_confirmed" + | "design_decided" + | "implementation_started" + | "issue_resolved" + | "milestone_reached" + | "other"; + summary: string; + timestamp: string; // ISO string +}; + +export type MemoryKind = "rule" | "preference" | "fact" | "task" | "ephemeral"; + +export type CandidateMemory = { + kind: MemoryKind; + /** Optional canonical slot/key for conflict resolution (e.g. preference:timezone). */ + memoryKey?: string; + /** Optional short subject label (best-effort). */ + subject?: string; + /** Optional ISO expiration timestamp for ephemeral memories. */ + expiresAt?: string; + /** 0..1 confidence (heuristic, best-effort). */ + confidence?: number; + content: string; + importance: number; + entities: string[]; + topics: string[]; + createdAt: string; // ISO +}; + +export type CandidateMemoryDraft = { + kind?: MemoryKind; + memoryKey?: string; + subject?: string; + expiresAt?: string; + confidence?: number; + content: string; + entities: string[]; + topics: string[]; + createdAt: string; // ISO + signals: { + frequency: number; + user_intent: number; // 0..1 + length: number; + }; +}; diff --git a/packages/deep-memory-server/src/updater.test.ts b/packages/deep-memory-server/src/updater.test.ts new file mode 100644 index 0000000000000..0565fb86de428 --- /dev/null +++ b/packages/deep-memory-server/src/updater.test.ts @@ -0,0 +1,215 @@ +import { describe, expect, it } from "vitest"; +import { DeepMemoryUpdater } from "./updater.js"; + +describe("DeepMemoryUpdater", () => { + it("filters sensitive + low-importance drafts and processes important ones", async () => { + const analyzer = { + analyze: () => ({ + entities: [], + topics: [], + events: [], + drafts: [ + { + content: "password=supersecret", + entities: [], + topics: [], + createdAt: new Date().toISOString(), + signals: { frequency: 10, user_intent: 1, length: 50 }, + }, + { + content: "duplicate but low importance", + entities: [], + topics: [], + createdAt: new Date().toISOString(), + signals: { frequency: 1, user_intent: 0.2, length: 50 }, + }, + { + content: "important memory that should be stored", + entities: ["ProjectX"], + topics: ["deep-memory"], + createdAt: new Date().toISOString(), + signals: { frequency: 2, user_intent: 0.9, length: 120 }, + }, + { + content: "very important duplicate that should reuse existing id", + entities: ["ProjectX"], + topics: ["deep-memory"], + createdAt: new Date().toISOString(), + signals: { frequency: 10, user_intent: 1, length: 2000 }, + }, + ], + filtered: { added: 4, filtered: 0 }, + }), + }; + + const embedder = { embed: async () => [0, 0, 0] }; + + const qdrantUpserts: Array<{ id: string; payload: unknown }> = []; + let searchCalls = 0; + const qdrant = { + search: async ({ limit }: { limit: number }) => { + // first call per draft does (limit=1) novelty+dedupe + // later we might do RELATED_TO search (limit>1), but this test doesn't rely on it + if (limit !== 1) { + return []; + } + searchCalls += 1; + // Calls happen only for non-sensitive drafts. + // 1) low importance: high similarity => low novelty + // 2) important: low similarity => high novelty + // 3) duplicate: very high similarity => dedupe to existing id + if (searchCalls === 1) { + return [{ id: "mem_existing", score: 0.95, payload: undefined }]; + } + if (searchCalls === 2) { + return [{ id: "mem_other", score: 0.2, payload: undefined }]; + } + return [{ id: "mem_existing", score: 0.95, payload: undefined }]; + }, + getMemory: async () => ({ + id: "default::mem_existing", + payload: { + id: "default::mem_existing", + namespace: "default", + content: "old", + session_id: "s0", + created_at: "2020-01-01T00:00:00.000Z", + updated_at: "2020-01-01T00:00:00.000Z", + importance: 0.6, + frequency: 3, + entities: ["ProjectX"], + topics: ["deep-memory"], + }, + }), + upsertMemory: async ({ id, payload }: { id: string; payload: unknown }) => { + qdrantUpserts.push({ id, payload }); + }, + }; + + const neo4jMemories: string[] = []; + const neo4j = { + upsertSession: async () => {}, + getSessionIngestMeta: async () => ({}), + setSessionIngestMeta: async () => {}, + upsertTopic: async () => {}, + linkSessionTopic: async () => {}, + upsertEntity: async () => {}, + linkTopicEntity: async () => {}, + upsertEvent: async () => {}, + eventId: () => "e1", + linkSessionEvent: async () => {}, + linkEventTopic: async () => {}, + linkEventEntity: async () => {}, + upsertMemory: async ({ id }: { id: string }) => { + neo4jMemories.push(id); + }, + linkMemoryTopic: async () => {}, + linkMemoryEntity: async () => {}, + linkMemoryRelated: async () => {}, + }; + + const updater = new DeepMemoryUpdater({ + analyzer: analyzer as unknown as never, + embedder: embedder as unknown as never, + qdrant: qdrant as unknown as never, + neo4j: neo4j as unknown as never, + minSemanticScore: 0.6, + importanceThreshold: 0.5, + maxMemoriesPerUpdate: 20, + dedupeScore: 0.92, + relatedTopK: 0, + sensitiveFilterEnabled: true, + }); + + const out = await updater.update({ namespace: "default", sessionId: "s1", messages: [] }); + + // We should store 2 memories: the "important" one and the "very important duplicate". + expect(out.memories_added).toBe(2); + expect(neo4jMemories.length).toBe(2); + + // One of the upserts should target the deduped existing id. + const ids = qdrantUpserts.map((u) => u.id); + expect(ids).toContain("default::mem_existing"); + + const existingPayload = qdrantUpserts.find((u) => u.id === "default::mem_existing")! + .payload as Record; + expect(existingPayload.namespace).toBe("default"); + expect(existingPayload.frequency).toBe(4); + }); + + it("can optionally return memory ids for traceability", async () => { + const analyzer = { + analyze: () => ({ + entities: [], + topics: [], + events: [], + drafts: [ + { + content: "important memory that should be stored", + entities: [], + topics: [], + createdAt: new Date().toISOString(), + signals: { frequency: 2, user_intent: 1, length: 120 }, + }, + { + content: "another important memory", + entities: [], + topics: [], + createdAt: new Date().toISOString(), + signals: { frequency: 2, user_intent: 1, length: 120 }, + }, + ], + filtered: { added: 2, filtered: 0 }, + }), + }; + const embedder = { embed: async () => [0, 0, 0] }; + const qdrant = { + search: async () => [], + getMemory: async () => null, + upsertMemory: async () => {}, + }; + const neo4j = { + upsertSession: async () => {}, + getSessionIngestMeta: async () => ({}), + setSessionIngestMeta: async () => {}, + upsertTopic: async () => {}, + linkSessionTopic: async () => {}, + upsertEntity: async () => {}, + linkTopicEntity: async () => {}, + upsertEvent: async () => {}, + eventId: () => "e1", + linkSessionEvent: async () => {}, + linkEventTopic: async () => {}, + linkEventEntity: async () => {}, + upsertMemory: async () => {}, + linkMemoryTopic: async () => {}, + linkMemoryEntity: async () => {}, + linkMemoryRelated: async () => {}, + }; + + const updater = new DeepMemoryUpdater({ + analyzer: analyzer as unknown as never, + embedder: embedder as unknown as never, + qdrant: qdrant as unknown as never, + neo4j: neo4j as unknown as never, + minSemanticScore: 0, + importanceThreshold: 0.1, + maxMemoriesPerUpdate: 20, + dedupeScore: 0.92, + relatedTopK: 0, + sensitiveFilterEnabled: false, + }); + + const out = await updater.update({ + namespace: "default", + sessionId: "s-trace", + messages: [], + returnMemoryIds: { max: 1 }, + }); + + expect(out.status).toBe("processed"); + expect(Array.isArray((out as Record).memory_ids)).toBe(true); + expect(((out as Record).memory_ids as unknown[]).length).toBe(1); + expect((out as Record).memory_ids_truncated).toBe(true); + }); +}); diff --git a/packages/deep-memory-server/src/updater.ts b/packages/deep-memory-server/src/updater.ts new file mode 100644 index 0000000000000..1b28ab22c0748 --- /dev/null +++ b/packages/deep-memory-server/src/updater.ts @@ -0,0 +1,357 @@ +import crypto from "node:crypto"; +import { SessionAnalyzer } from "./analyzer.js"; +import { EmbeddingModel } from "./embeddings.js"; +import { computeImportance } from "./importance.js"; +import { Neo4jStore } from "./neo4j.js"; +import { QdrantStore, type QdrantMemoryPayload } from "./qdrant.js"; +import type { SensitiveFilter } from "./safety.js"; +import { createSensitiveFilter } from "./safety.js"; +import type { UpdateMemoryIndexResponse } from "./types.js"; +import { clamp, stableHash } from "./utils.js"; + +export class DeepMemoryUpdater { + private readonly analyzer: SessionAnalyzer; + private readonly embedder: EmbeddingModel; + private readonly qdrant: QdrantStore; + private readonly neo4j: Neo4jStore; + private readonly minSemanticScore: number; + private readonly importanceThreshold: number; + private readonly maxMemoriesPerUpdate: number; + private readonly dedupeScore: number; + private readonly relatedTopK: number; + private readonly sensitiveFilterEnabled: boolean; + private readonly sensitiveFilter: SensitiveFilter; + + constructor(params: { + analyzer: SessionAnalyzer; + embedder: EmbeddingModel; + qdrant: QdrantStore; + neo4j: Neo4jStore; + minSemanticScore: number; + importanceThreshold: number; + maxMemoriesPerUpdate: number; + dedupeScore: number; + relatedTopK: number; + sensitiveFilterEnabled: boolean; + sensitiveFilter?: SensitiveFilter; + }) { + this.analyzer = params.analyzer; + this.embedder = params.embedder; + this.qdrant = params.qdrant; + this.neo4j = params.neo4j; + this.minSemanticScore = params.minSemanticScore; + this.importanceThreshold = params.importanceThreshold; + this.maxMemoriesPerUpdate = params.maxMemoriesPerUpdate; + this.dedupeScore = params.dedupeScore; + this.relatedTopK = Math.max(0, params.relatedTopK); + this.sensitiveFilterEnabled = params.sensitiveFilterEnabled; + this.sensitiveFilter = + params.sensitiveFilter ?? + createSensitiveFilter({ + SENSITIVE_RULESET_VERSION: "builtin-v1", + SENSITIVE_ALLOW_REGEX_JSON: undefined, + SENSITIVE_DENY_REGEX_JSON: undefined, + }); + } + + async update(params: { + namespace: string; + sessionId: string; + messages: unknown[]; + returnMemoryIds?: { max: number }; + }): Promise { + const messageCount = Array.isArray(params.messages) ? params.messages.length : 0; + const transcriptHash = crypto + .createHash("sha256") + .update(JSON.stringify(params.messages ?? [])) + .digest("hex"); + + // Ensure Session exists (needed for idempotency meta). + await this.neo4j.upsertSession({ namespace: params.namespace, sessionId: params.sessionId }); + try { + const meta = await this.neo4j.getSessionIngestMeta({ + namespace: params.namespace, + sessionId: params.sessionId, + }); + if (meta.transcriptHash === transcriptHash) { + // Idempotency: transcript already ingested. Treat as successfully processed. + return { status: "processed", memories_added: 0, memories_filtered: 0 }; + } + } catch { + // If Neo4j is unavailable, we proceed best-effort (may duplicate). + } + + const analysis = this.analyzer.analyze({ + sessionId: params.sessionId, + messages: params.messages, + maxMemoriesPerSession: this.maxMemoriesPerUpdate, + importanceThreshold: this.importanceThreshold, + }); + + for (const t of analysis.topics) { + await this.neo4j.upsertTopic({ namespace: params.namespace, topic: t }); + await this.neo4j.linkSessionTopic({ + namespace: params.namespace, + sessionId: params.sessionId, + topicName: t.name, + }); + } + for (const e of analysis.entities) { + await this.neo4j.upsertEntity({ namespace: params.namespace, entity: e }); + for (const t of analysis.topics.slice(0, 5)) { + await this.neo4j.linkTopicEntity({ + namespace: params.namespace, + topicName: t.name, + entityName: e.name, + entityType: e.type, + }); + } + } + for (const ev of analysis.events) { + await this.neo4j.upsertEvent({ namespace: params.namespace, event: ev }); + const eventId = this.neo4j.eventId({ namespace: params.namespace, event: ev }); + await this.neo4j.linkSessionEvent({ + namespace: params.namespace, + sessionId: params.sessionId, + eventId, + }); + for (const t of analysis.topics.slice(0, 3)) { + await this.neo4j.linkEventTopic({ + namespace: params.namespace, + eventId, + topicName: t.name, + }); + } + const topEntities = analysis.entities.slice(0, 3); + for (const ent of topEntities) { + await this.neo4j.linkEventEntity({ + namespace: params.namespace, + eventId, + entityName: ent.name, + entityType: ent.type, + }); + } + } + + let added = 0; + let filtered = analysis.filtered.filtered; + const requestedMaxIds = Math.max(0, Math.trunc(params.returnMemoryIds?.max ?? 0)); + const memoryIds = requestedMaxIds > 0 ? new Set() : null; + const entityTypeByName = new Map(); + for (const e of analysis.entities) { + entityTypeByName.set(e.name, e.type); + } + const nowIso = new Date().toISOString(); + for (const draft of analysis.drafts) { + if (this.sensitiveFilterEnabled) { + const det = this.sensitiveFilter.detect(draft.content); + if (det.sensitive) { + filtered += 1; + continue; + } + } + + const vec = await this.embedder.embed(draft.content); + + // Novelty + global dedupe (best-effort). If Qdrant is down we fall back to session hash. + let bestId: string | null = null; + let bestScore = 0; + try { + const top = await this.qdrant.search({ + vector: vec, + limit: 1, + minScore: 0, + namespace: params.namespace, + }); + const best = top[0]; + if (best?.id) { + bestId = best.id; + bestScore = best.score ?? 0; + } + } catch { + // ignore + } + + const novelty = clamp(1 - bestScore, 0, 1); + const importance = computeImportance({ + frequency: draft.signals.frequency, + novelty, + user_intent: draft.signals.user_intent, + length: draft.signals.length, + }); + if (importance < this.importanceThreshold) { + filtered += 1; + continue; + } + + const isDuplicate = bestId && bestScore >= this.dedupeScore; + const rawId = isDuplicate + ? bestId! + : `mem_${stableHash(`${params.sessionId}:${draft.content}`)}`; + const id = rawId.includes("::") ? rawId : `${params.namespace}::${rawId}`; + + const mergedEntities = new Set(draft.entities); + const mergedTopics = new Set(draft.topics); + let mergedImportance = importance; + let mergedFrequency = 1; + if (isDuplicate) { + try { + const existing = await this.qdrant.getMemory(id); + const p = existing?.payload; + if (p?.entities) { + p.entities.forEach((e) => mergedEntities.add(e)); + } + if (p?.topics) { + p.topics.forEach((t) => mergedTopics.add(t)); + } + mergedImportance = Math.max(mergedImportance, p?.importance ?? 0); + mergedFrequency = (p?.frequency ?? 1) + 1; + // Preserve durable classification/slotting if present. + if (!draft.memoryKey && p?.memory_key) { + draft.memoryKey = p.memory_key; + } + if (!draft.subject && p?.subject) { + draft.subject = p.subject; + } + if (!draft.kind && typeof p?.kind === "string") { + const k = p.kind; + if ( + k === "rule" || + k === "preference" || + k === "fact" || + k === "task" || + k === "ephemeral" + ) { + draft.kind = k; + } + } + if (!draft.expiresAt && p?.expires_at) { + draft.expiresAt = p.expires_at; + } + if (typeof draft.confidence !== "number" && typeof p?.confidence === "number") { + draft.confidence = p.confidence; + } + } catch { + // ignore + } + } + + const mem = { + kind: draft.kind ?? "fact", + memoryKey: draft.memoryKey, + subject: draft.subject, + expiresAt: draft.expiresAt, + confidence: draft.confidence, + content: draft.content, + importance: mergedImportance, + entities: Array.from(mergedEntities).slice(0, 10), + topics: Array.from(mergedTopics).slice(0, 10), + createdAt: draft.createdAt, + }; + + await this.neo4j.upsertMemory({ + namespace: params.namespace, + id, + memory: mem, + sessionId: params.sessionId, + }); + for (const t of mem.topics) { + await this.neo4j.linkMemoryTopic({ + namespace: params.namespace, + memoryId: id, + topicName: t, + }); + } + for (const name of mem.entities) { + const type = entityTypeByName.get(name) ?? "other"; + await this.neo4j.linkMemoryEntity({ + namespace: params.namespace, + memoryId: id, + entityName: name, + entityType: type, + }); + } + + // Upsert to Qdrant (best-effort). + try { + const payload: QdrantMemoryPayload = { + id, + namespace: params.namespace, + kind: mem.kind, + memory_key: mem.memoryKey, + subject: mem.subject, + expires_at: mem.expiresAt, + confidence: mem.confidence, + content: mem.content, + session_id: params.sessionId, + source_transcript_hash: transcriptHash, + source_message_count: messageCount, + created_at: mem.createdAt, + updated_at: nowIso, + importance: mem.importance, + frequency: mergedFrequency, + entities: mem.entities, + topics: mem.topics, + }; + await this.qdrant.upsertMemory({ id, vector: vec, payload }); + } catch { + // ignore + } + + // Create synapse links: connect this memory to nearby ones (best-effort). + if (this.relatedTopK > 0) { + try { + const hits = await this.qdrant.search({ + vector: vec, + limit: Math.max(1, this.relatedTopK + 1), + minScore: Math.max(this.minSemanticScore, 0.8), + namespace: params.namespace, + }); + for (const hit of hits) { + if (!hit?.id || hit.id === id) { + continue; + } + await this.neo4j.linkMemoryRelated({ + namespace: params.namespace, + fromMemoryId: id, + toMemoryId: hit.id, + score: hit.score ?? 0, + }); + } + } catch { + // ignore + } + } + + added += 1; + if (memoryIds && memoryIds.size < requestedMaxIds) { + memoryIds.add(id); + } + } + + // Persist idempotency markers (best-effort). + try { + await this.neo4j.setSessionIngestMeta({ + namespace: params.namespace, + sessionId: params.sessionId, + transcriptHash, + messageCount, + }); + } catch { + // ignore + } + + const out: UpdateMemoryIndexResponse = { + status: "processed", + memories_added: added, + memories_filtered: filtered, + }; + if (memoryIds) { + const ids = Array.from(memoryIds); + (out as unknown as Record).memory_ids = ids; + (out as unknown as Record).memory_ids_truncated = + ids.length >= requestedMaxIds; + } + return out; + } +} diff --git a/packages/deep-memory-server/src/utils.ts b/packages/deep-memory-server/src/utils.ts new file mode 100644 index 0000000000000..689efa69587cc --- /dev/null +++ b/packages/deep-memory-server/src/utils.ts @@ -0,0 +1,20 @@ +export function clamp(value: number, min: number, max: number): number { + if (!Number.isFinite(value)) { + return min; + } + return Math.max(min, Math.min(max, value)); +} + +export function safeTrim(value: unknown): string { + return typeof value === "string" ? value.trim() : ""; +} + +export function stableHash(input: string): string { + // Cheap stable hash for ids; not cryptographic. + let h = 2166136261; + for (let i = 0; i < input.length; i += 1) { + h ^= input.charCodeAt(i); + h = Math.imul(h, 16777619); + } + return (h >>> 0).toString(16); +} diff --git a/packages/deep-memory-server/src/validate-reindex.ts b/packages/deep-memory-server/src/validate-reindex.ts new file mode 100644 index 0000000000000..3e84f088c06e6 --- /dev/null +++ b/packages/deep-memory-server/src/validate-reindex.ts @@ -0,0 +1,220 @@ +import fs from "node:fs/promises"; +import process from "node:process"; +import { loadConfig } from "./config.js"; +import { createLogger } from "./logger.js"; +import { Neo4jStore } from "./neo4j.js"; +import { QdrantStore } from "./qdrant.js"; +import { stableHash } from "./utils.js"; + +type Args = { + namespace?: string; + allNamespaces: boolean; + targetCollection?: string; + sampleSize: number; + outFile?: string; +}; + +function parseArgs(argv: string[]): Args { + const out: Args = { allNamespaces: false, sampleSize: 50 }; + for (let i = 0; i < argv.length; i += 1) { + const t = argv[i]?.trim(); + if (!t) { + continue; + } + if (t === "--namespace" || t === "-n") { + out.namespace = argv[i + 1]?.trim(); + i += 1; + continue; + } + if (t === "--all-namespaces" || t === "--all") { + out.allNamespaces = true; + continue; + } + if (t === "--target-collection" || t === "--collection") { + out.targetCollection = argv[i + 1]?.trim(); + i += 1; + continue; + } + if (t === "--sample-size") { + const v = Number(argv[i + 1]); + if (Number.isFinite(v) && v > 0) { + out.sampleSize = Math.max(1, Math.min(500, Math.floor(v))); + } + i += 1; + continue; + } + if (t === "--out") { + const v = argv[i + 1]?.trim(); + if (v) { + out.outFile = v; + } + i += 1; + continue; + } + if (t === "--help" || t === "-h") { + // eslint-disable-next-line no-console + console.log(` +Usage: pnpm --dir packages/deep-memory-server validate-reindex -- --namespace [options] + +Options: + --namespace, -n Validate a single namespace (recommended) + --all-namespaces, --all Validate all namespaces (can be slow) + --target-collection Qdrant collection to validate (default: QDRANT_COLLECTION) + --sample-size Sample size per namespace (default: 50, max: 500) + --out Write JSON report to file + --help, -h Show help +`); + process.exit(0); + } + } + if (!out.allNamespaces) { + out.namespace = out.namespace?.trim() || undefined; + if (!out.namespace) { + throw new Error("missing --namespace (or use --all-namespaces)"); + } + } + return out; +} + +type SampleItem = { id: string; content: string; namespace: string; hash: number }; + +function hashToNumber(hex: string): number { + const h = hex.slice(0, 8); + const n = Number.parseInt(h, 16); + return Number.isFinite(n) ? n : 0; +} + +function maybeAddSample(samples: SampleItem[], item: SampleItem, limit: number) { + if (samples.length < limit) { + samples.push(item); + samples.sort((a, b) => a.hash - b.hash); + return; + } + const worst = samples[samples.length - 1]; + if (!worst || item.hash >= worst.hash) { + return; + } + samples[samples.length - 1] = item; + samples.sort((a, b) => a.hash - b.hash); +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const cfg = loadConfig(); + const log = createLogger(cfg.LOG_LEVEL); + const targetCollection = (args.targetCollection ?? cfg.QDRANT_COLLECTION).trim(); + if (!targetCollection) { + throw new Error("empty target collection"); + } + + const neo4j = new Neo4jStore({ + uri: cfg.NEO4J_URI, + user: cfg.NEO4J_USER, + password: cfg.NEO4J_PASSWORD, + }); + const qdrant = new QdrantStore({ + url: cfg.QDRANT_URL, + apiKey: cfg.QDRANT_API_KEY, + collection: targetCollection, + dims: cfg.VECTOR_DIMS, + }); + + const namespaces = args.allNamespaces + ? await neo4j.listNamespaces({ limit: 10_000 }) + : [args.namespace!]; + + const report: { + ok: boolean; + targetCollection: string; + namespaces: Array<{ + namespace: string; + neo4jCount: number; + qdrantCount: number; + sampleSize: number; + sampleHits: number; + missingIds: string[]; + }>; + finishedAt: string; + } = { ok: true, targetCollection, namespaces: [], finishedAt: "" }; + + try { + for (const ns of namespaces) { + const neo4jCount = await neo4j.countMemories({ namespace: ns }); + const qdrantCount = await qdrant.countMemories({ namespace: ns }); + + const samples: SampleItem[] = []; + let afterId: string | undefined; + while (true) { + const page = await neo4j.scanMemories({ namespace: ns, afterId, limit: 200 }); + if (page.length === 0) { + break; + } + for (const m of page) { + afterId = m.id; + const h = hashToNumber(stableHash(m.id)); + maybeAddSample( + samples, + { id: m.id, content: m.content, namespace: ns, hash: h }, + args.sampleSize, + ); + } + } + + const missingIds: string[] = []; + let hits = 0; + for (const s of samples) { + const hit = await qdrant.getMemory(s.id); + if (!hit?.payload) { + missingIds.push(s.id); + continue; + } + if (hit.payload.namespace !== ns) { + missingIds.push(s.id); + continue; + } + if (hit.payload.content !== s.content) { + missingIds.push(s.id); + continue; + } + hits += 1; + } + + const ok = missingIds.length === 0; + report.ok = report.ok && ok; + report.namespaces.push({ + namespace: ns, + neo4jCount, + qdrantCount, + sampleSize: samples.length, + sampleHits: hits, + missingIds: missingIds.slice(0, 50), + }); + log.info( + { + namespace: ns, + neo4jCount, + qdrantCount, + sample: samples.length, + missing: missingIds.length, + }, + "validate-reindex namespace summary", + ); + } + } finally { + await neo4j.close().catch(() => {}); + } + + report.finishedAt = new Date().toISOString(); + if (args.outFile?.trim()) { + await fs.writeFile(args.outFile.trim(), `${JSON.stringify(report, null, 2)}\n`, "utf8"); + } + // eslint-disable-next-line no-console + console.log(`validate-reindex: ok=${report.ok} namespaces=${report.namespaces.length}`); + process.exit(report.ok ? 0 : 2); +} + +void main().catch((err) => { + // eslint-disable-next-line no-console + console.error(String(err instanceof Error ? (err.stack ?? err.message) : err)); + process.exit(1); +}); diff --git a/packages/deep-memory-server/tsconfig.json b/packages/deep-memory-server/tsconfig.json new file mode 100644 index 0000000000000..45d9c89a8d5f6 --- /dev/null +++ b/packages/deep-memory-server/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "noEmit": false, + "allowImportingTsExtensions": false, + "composite": false, + "declaration": false, + "sourceMap": true + }, + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.test.ts"] +} diff --git a/packages/deep-memory-server/vitest.config.ts b/packages/deep-memory-server/vitest.config.ts new file mode 100644 index 0000000000000..336173994fd3d --- /dev/null +++ b/packages/deep-memory-server/vitest.config.ts @@ -0,0 +1,16 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { defineConfig } from "vitest/config"; + +const pkgRoot = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(pkgRoot, "../.."); + +export default defineConfig({ + test: { + include: ["src/**/*.test.ts"], + exclude: ["dist/**", "**/node_modules/**"], + setupFiles: [path.join(repoRoot, "test", "setup.ts")], + testTimeout: 60_000, + hookTimeout: 60_000, + }, +}); diff --git a/packages/rustfs-worker/Dockerfile b/packages/rustfs-worker/Dockerfile new file mode 100644 index 0000000000000..6ee497d7c18f8 --- /dev/null +++ b/packages/rustfs-worker/Dockerfile @@ -0,0 +1,34 @@ +FROM node:22-slim AS deps +WORKDIR /app + +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml .npmrc ./ +COPY packages/rustfs-worker/package.json packages/rustfs-worker/package.json + +RUN corepack enable && pnpm install --frozen-lockfile --filter @openclaw/rustfs-worker... + +FROM node:22-slim AS build +WORKDIR /app +RUN corepack enable +COPY --from=deps /app/node_modules ./node_modules +COPY --from=deps /app/packages/rustfs-worker/node_modules ./packages/rustfs-worker/node_modules +COPY --from=deps /app/pnpm-lock.yaml ./pnpm-lock.yaml +COPY --from=deps /app/package.json ./package.json +COPY --from=deps /app/pnpm-workspace.yaml ./pnpm-workspace.yaml +COPY --from=deps /app/.npmrc ./.npmrc + +COPY tsconfig.json ./tsconfig.json +COPY packages/rustfs-worker/tsconfig.json packages/rustfs-worker/tsconfig.json +COPY packages/rustfs-worker/package.json packages/rustfs-worker/package.json +COPY packages/rustfs-worker/src packages/rustfs-worker/src + +RUN pnpm --dir packages/rustfs-worker run build + +FROM node:22-slim AS runtime +WORKDIR /app +ENV NODE_ENV=production +COPY --from=build /app/node_modules ./node_modules +COPY --from=build /app/packages/rustfs-worker/node_modules ./packages/rustfs-worker/node_modules +COPY --from=build /app/packages/rustfs-worker/dist ./packages/rustfs-worker/dist +COPY --from=build /app/packages/rustfs-worker/package.json ./packages/rustfs-worker/package.json + +CMD ["node", "packages/rustfs-worker/dist/index.js"] diff --git a/packages/rustfs-worker/README.md b/packages/rustfs-worker/README.md new file mode 100644 index 0000000000000..666a358097b5d --- /dev/null +++ b/packages/rustfs-worker/README.md @@ -0,0 +1,55 @@ +# RustFS Worker + +一个最小可用的后台 worker:通过 RustFS 的原子 `claim_extract`(租约领取)接口拉取待处理文件,做抽取/切片后: + +- 回写 RustFS `annotations` +- 更新 RustFS `extract_status`(`indexed` / `skipped` / `failed`) +- 调用 deep-memory-server 的 `/update_memory_index`,将文件内容(分 segment)纳入语义记忆系统(可选返回 `memory_ids` 用于追溯) + +当前支持的抽取类型(MVP): + +- 纯文本:`text/*`、`md/json/yaml/xml/toml/jsonl/txt` +- HTML:`.html/.htm`(title + heading → section segments) +- 代码:`.ts/.js/.py/.go/.rs/.java/.kt/.swift/.sh/.css`(按函数/类/结构边界做粗切分) +- PDF:`.pdf`(按 page → segments) +- DOCX:`.docx`(按段落 → segments) +- PPTX:`.pptx`(按 slide → segments) + +## 环境变量 + +- `RUSTFS_BASE_URL`(必填) +- `RUSTFS_API_KEY`(可选;RustFS 若要求鉴权则需要) +- `RUSTFS_TENANT_ID`(可选;仅用于调试/覆盖 tenant,正常依赖 api key 绑定的 tenant) +- `RUSTFS_POLL_INTERVAL_MS`(默认 `5000`) +- `RUSTFS_LEASE_MS`(默认 `300000`):处理中的任务超过该时间会被重新投放为可领取 +- `RUSTFS_PENDING_LIMIT`(默认 `25`) +- `RUSTFS_MAX_DOWNLOAD_BYTES`(默认 `2097152`) + +- `DEEP_MEMORY_BASE_URL`(必填) +- `DEEP_MEMORY_API_KEY`(可选;deep-memory-server 若要求鉴权则需要) +- `DEEP_MEMORY_NAMESPACE`(可选) +- `DEEP_MEMORY_ASYNC`(默认 `true`) +- `DEEP_MEMORY_TIMEOUT_MS`(默认 `30000`) +- `DEEP_MEMORY_CHUNK_MAX_CHARS`(默认 `3500`) +- `DEEP_MEMORY_CHUNK_OVERLAP_CHARS`(默认 `200`) +- `DEEP_MEMORY_BACKOFF_MS`(默认 `10000`):deep-memory 过载时退避等待 +- `DEEP_MEMORY_RETURN_MEMORY_IDS`(默认 `false`):请求 deep-memory 返回本次写入/更新的 `memory_ids`(**仅在 `DEEP_MEMORY_ASYNC=false` 时生效**) +- `DEEP_MEMORY_MAX_RETURN_MEMORY_IDS`(默认 `200`,最大 `1000`):返回 `memory_ids` 的上限 +- `DEEP_MEMORY_INSPECT_ENABLED`(默认 `false`):indexed 后调用 deep-memory `POST /session/inspect`,回写 `annotations.semantics` +- `DEEP_MEMORY_INSPECT_LIMIT`(默认 `100`):inspect 拉取 memories 上限 +- `DEEP_MEMORY_INSPECT_INCLUDE_CONTENT`(默认 `false`):inspect 时是否包含代表性内容(用于生成 summary) + +## 运行 + +本地开发: + +```bash +pnpm -w --filter @openclaw/rustfs-worker... dev +``` + +生产(build 后运行): + +```bash +pnpm -w --filter @openclaw/rustfs-worker... build +pnpm -w --filter @openclaw/rustfs-worker... start +``` diff --git a/packages/rustfs-worker/package.json b/packages/rustfs-worker/package.json new file mode 100644 index 0000000000000..daccad65d9b69 --- /dev/null +++ b/packages/rustfs-worker/package.json @@ -0,0 +1,28 @@ +{ + "name": "@openclaw/rustfs-worker", + "version": "0.1.0", + "private": true, + "type": "module", + "main": "dist/index.js", + "scripts": { + "build": "tsc -p tsconfig.json", + "dev": "node --import tsx src/index.ts", + "start": "node dist/index.js" + }, + "dependencies": { + "jszip": "^3.10.1", + "mammoth": "^1.11.0", + "pdf-parse": "^2.4.5", + "pino": "^9.6.0", + "zod": "^4.3.6" + }, + "devDependencies": { + "@types/node": "^25.2.0", + "@types/pdf-parse": "^1.1.5", + "tsx": "^4.21.0", + "typescript": "^5.9.3" + }, + "engines": { + "node": ">=22.12.0" + } +} diff --git a/packages/rustfs-worker/src/index.ts b/packages/rustfs-worker/src/index.ts new file mode 100644 index 0000000000000..e7790936ecec9 --- /dev/null +++ b/packages/rustfs-worker/src/index.ts @@ -0,0 +1,1581 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname } from "node:path"; +import JSZip from "jszip"; +import mammoth from "mammoth"; +import { PDFParse } from "pdf-parse"; +import pino from "pino"; +import { z } from "zod"; + +type RustFsFileMeta = { + file_id: string; + tenant_id: string; + session_id?: string | null; + filename: string; + mime?: string | null; + size: number; + sha256: string; + created_at_ms: number; + source?: string | null; + encrypted: boolean; + extract_status?: string | null; + extract_updated_at_ms?: number | null; + extract_attempt?: number | null; + extract_error?: string | null; + // annotations is opaque json (optional) + annotations?: unknown; +}; + +type ExtractionSegmentV1 = { + text: string; + locator?: { + kind: "chunk" | "page" | "slide" | "section" | "unknown"; + index: number; // 1-based + total: number; + startChar?: number; + endChar?: number; + label?: string; + }; +}; + +type ExtractionResultV1 = { + schema_version: 1; + docTypeGuess: string; + languageGuess?: string; + segments: ExtractionSegmentV1[]; + warnings: string[]; + stats: { + bytes: number; + chars: number; + segments: number; + truncated: boolean; + }; +}; + +function buildAnnotationsV1(params: { + file: RustFsFileMeta; + existingAnnotations?: unknown; + deepMemory: { + baseUrl: string; + namespace?: string; + sessionId: string; + updateResponse?: unknown; + error?: string; + overloaded?: boolean; + memoryIds?: string[]; + memoryIdsTruncated?: boolean; + }; + extraction: ExtractionResultV1; + semantics?: { + topics?: string[]; + entities?: string[]; + summary?: string; + }; +}): Record { + const base = + params.existingAnnotations && typeof params.existingAnnotations === "object" + ? { ...(params.existingAnnotations as Record) } + : ({} as Record); + + const ingestRaw = base.openclaw_ingest; + const ingest = + ingestRaw && typeof ingestRaw === "object" && !Array.isArray(ingestRaw) + ? (ingestRaw as Record) + : null; + const ingestKind = ingest && typeof ingest.kind === "string" ? ingest.kind.trim() : ""; + const ingestHint = ingest && typeof ingest.hint === "string" ? ingest.hint.trim() : ""; + const ingestTags = + ingest && Array.isArray(ingest.tags) + ? ingest.tags + .filter((t): t is string => typeof t === "string") + .map((t) => t.trim()) + .filter(Boolean) + .slice(0, 50) + : undefined; + + return { + ...base, + schema_version: 1, + classification: { + kind: ingestKind || undefined, + hint: ingestHint || undefined, + tags: ingestTags && ingestTags.length > 0 ? ingestTags : undefined, + }, + semantics: { + topics: params.semantics?.topics, + entities: params.semantics?.entities, + summary: params.semantics?.summary, + updated_at_ms: + params.semantics && + ((params.semantics.topics && params.semantics.topics.length > 0) || + (params.semantics.entities && params.semantics.entities.length > 0) || + (params.semantics.summary && params.semantics.summary.length > 0)) + ? Date.now() + : undefined, + source: + params.semantics && + ((params.semantics.topics && params.semantics.topics.length > 0) || + (params.semantics.entities && params.semantics.entities.length > 0) || + (params.semantics.summary && params.semantics.summary.length > 0)) + ? "deep-memory" + : undefined, + }, + // Keep legacy fields for compatibility; downstream can migrate gradually. + rustfs_worker: { + version: 1, + indexed_at_ms: Date.now(), + deep_memory: { + base_url: params.deepMemory.baseUrl, + namespace: params.deepMemory.namespace, + session_id: params.deepMemory.sessionId, + update_response: params.deepMemory.updateResponse, + error: params.deepMemory.error, + overloaded: params.deepMemory.overloaded, + }, + extract: { + mime: params.file.mime ?? undefined, + bytes: params.extraction.stats.bytes, + truncated: params.extraction.stats.truncated, + segments: params.extraction.segments.length, + doc_type_guess: params.extraction.docTypeGuess, + language_guess: params.extraction.languageGuess, + warnings: params.extraction.warnings, + }, + }, + extraction: { + status: params.deepMemory.error ? "error" : "indexed", + extractor: `rustfs-worker:${params.extraction.docTypeGuess}:v1`, + attempt: params.file.extract_attempt ?? undefined, + last_error: params.deepMemory.error ?? undefined, + warnings: params.extraction.warnings, + doc_type_guess: params.extraction.docTypeGuess, + language_guess: params.extraction.languageGuess, + segments: { + count: params.extraction.segments.length, + }, + stats: params.extraction.stats, + }, + deep_memory: { + session_id: params.deepMemory.sessionId, + namespace: params.deepMemory.namespace, + // Optional future fields: memory_ids/source_ref mappings. + memory_ids: params.deepMemory.memoryIds, + memory_ids_truncated: params.deepMemory.memoryIdsTruncated, + }, + }; +} + +const EnvSchema = z.object({ + RUSTFS_BASE_URL: z.string().min(1), + RUSTFS_API_KEY: z.string().optional(), + RUSTFS_TENANT_ID: z.string().optional(), + RUSTFS_POLL_INTERVAL_MS: z.coerce.number().int().positive().default(5_000), + RUSTFS_LEASE_MS: z.coerce.number().int().positive().default(300_000), + RUSTFS_PENDING_LIMIT: z.coerce.number().int().positive().max(200).default(25), + RUSTFS_MAX_DOWNLOAD_BYTES: z.coerce + .number() + .int() + .positive() + .default(2 * 1024 * 1024), + RUSTFS_TOMBSTONE_POLL_INTERVAL_MS: z.coerce.number().int().positive().default(30_000), + RUSTFS_TOMBSTONE_LIMIT: z.coerce.number().int().positive().max(200).default(100), + RUSTFS_TOMBSTONE_SINCE_MS: z.coerce.number().int().nonnegative().default(0), + WORKER_STATE_PATH: z.string().optional(), + + DEEP_MEMORY_BASE_URL: z.string().min(1), + DEEP_MEMORY_API_KEY: z.string().optional(), + DEEP_MEMORY_NAMESPACE: z.string().optional(), + DEEP_MEMORY_ASYNC: z.coerce.boolean().default(true), + DEEP_MEMORY_TIMEOUT_MS: z.coerce.number().int().positive().default(30_000), + + // Traceability: request memory ids (only works when DEEP_MEMORY_ASYNC=false). + DEEP_MEMORY_RETURN_MEMORY_IDS: z.coerce.boolean().default(false), + DEEP_MEMORY_MAX_RETURN_MEMORY_IDS: z.coerce.number().int().positive().max(1000).default(200), + + // Chunking: convert extracted text into multiple messages for better semantic indexing. + DEEP_MEMORY_CHUNK_MAX_CHARS: z.coerce.number().int().positive().default(3500), + DEEP_MEMORY_CHUNK_OVERLAP_CHARS: z.coerce.number().int().nonnegative().default(200), + + // Backoff when deep-memory is overloaded. + DEEP_MEMORY_BACKOFF_MS: z.coerce.number().int().positive().default(10_000), + + // Optional: after indexing, ask deep-memory for session-level semantics (topics/entities/summary). + DEEP_MEMORY_INSPECT_ENABLED: z.coerce.boolean().default(false), + DEEP_MEMORY_INSPECT_LIMIT: z.coerce.number().int().positive().max(1000).default(100), + DEEP_MEMORY_INSPECT_INCLUDE_CONTENT: z.coerce.boolean().default(false), +}); + +function readStringArray(value: unknown, max: number): string[] | undefined { + if (!Array.isArray(value)) { + return undefined; + } + const out = value + .filter((v): v is string => typeof v === "string") + .map((s) => s.trim()) + .filter(Boolean) + .slice(0, Math.max(0, Math.min(1000, Math.trunc(max)))); + return out.length > 0 ? out : undefined; +} + +function extractMemoryIdsFromUpdateResponse(updateRes: unknown): { + ids?: string[]; + truncated?: boolean; +} { + if (!updateRes || typeof updateRes !== "object") { + return {}; + } + const r = updateRes as Record; + const ids = readStringArray(r.memory_ids, 1000); + const truncated = + typeof r.memory_ids_truncated === "boolean" ? r.memory_ids_truncated : undefined; + return { ids, truncated }; +} + +function extractSessionSemanticsFromInspectResponse(inspectRes: unknown): { + topics?: string[]; + entities?: string[]; + summary?: string; +} { + if (!inspectRes || typeof inspectRes !== "object") { + return {}; + } + const r = inspectRes as Record; + const topicsRaw = r.topics; + const entitiesRaw = r.entities; + const topics = + Array.isArray(topicsRaw) && topicsRaw.every((t) => t && typeof t === "object") + ? (topicsRaw as Array>) + .map((t) => (typeof t.name === "string" ? t.name.trim() : "")) + .filter(Boolean) + .slice(0, 20) + : undefined; + const entities = + Array.isArray(entitiesRaw) && entitiesRaw.every((t) => t && typeof t === "object") + ? (entitiesRaw as Array>) + .map((t) => (typeof t.name === "string" ? t.name.trim() : "")) + .filter(Boolean) + .slice(0, 20) + : undefined; + const summary = typeof r.summary === "string" ? r.summary.trim().slice(0, 1200) : undefined; + return { + topics: topics && topics.length > 0 ? topics : undefined, + entities: entities && entities.length > 0 ? entities : undefined, + summary: summary && summary.length > 0 ? summary : undefined, + }; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function normalizeBaseUrl(url: string): string { + return url.replace(/\/+$/, ""); +} + +function isSupportedTextLike(mime: string | undefined, filename: string): boolean { + const m = (mime ?? "").toLowerCase().trim(); + if (m.startsWith("text/")) { + return true; + } + if ( + m === "application/json" || + m === "application/xml" || + m === "application/yaml" || + m === "application/x-yaml" || + m === "application/toml" || + m === "application/x-ndjson" + ) { + return true; + } + const lower = filename.toLowerCase(); + return ( + lower.endsWith(".txt") || + lower.endsWith(".md") || + lower.endsWith(".markdown") || + lower.endsWith(".json") || + lower.endsWith(".jsonl") || + lower.endsWith(".xml") || + lower.endsWith(".yaml") || + lower.endsWith(".yml") || + lower.endsWith(".toml") + ); +} + +function isHtmlLike(mime: string | undefined, filename: string): boolean { + const m = (mime ?? "").toLowerCase().trim(); + if (m === "text/html" || m === "application/xhtml+xml") { + return true; + } + const lower = filename.toLowerCase(); + return lower.endsWith(".html") || lower.endsWith(".htm") || lower.endsWith(".xhtml"); +} + +function isCodeLike(_mime: string | undefined, filename: string): boolean { + const lang = guessLanguage(filename); + if (!lang) { + return false; + } + return [ + "typescript", + "javascript", + "python", + "go", + "rust", + "java", + "kotlin", + "swift", + "css", + "shell", + ].includes(lang); +} + +function isPdfLike(mime: string | undefined, filename: string): boolean { + const m = (mime ?? "").toLowerCase().trim(); + if (m === "application/pdf") { + return true; + } + return filename.toLowerCase().endsWith(".pdf"); +} + +function isDocxLike(mime: string | undefined, filename: string): boolean { + const m = (mime ?? "").toLowerCase().trim(); + if (m === "application/vnd.openxmlformats-officedocument.wordprocessingml.document") { + return true; + } + return filename.toLowerCase().endsWith(".docx"); +} + +function isPptxLike(mime: string | undefined, filename: string): boolean { + const m = (mime ?? "").toLowerCase().trim(); + if (m === "application/vnd.openxmlformats-officedocument.presentationml.presentation") { + return true; + } + return filename.toLowerCase().endsWith(".pptx"); +} + +function decodeHtmlEntities(input: string): string { + return input + .replaceAll(" ", " ") + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll(""", '"') + .replaceAll("'", "'") + .replace(/&#(\d+);/g, (_m, n) => { + const code = Number(n); + return Number.isFinite(code) ? String.fromCodePoint(code) : ""; + }); +} + +function extractHtmlLike(params: { + file: RustFsFileMeta; + html: string; + bytes: number; + truncated: boolean; + chunkMaxChars: number; + chunkOverlapChars: number; +}): ExtractionResultV1 { + const warnings: string[] = []; + const raw = params.html; + const titleMatch = raw.match(/]*>([\s\S]*?)<\/title>/i); + const title = titleMatch ? decodeHtmlEntities(titleMatch[1]).replace(/\s+/g, " ").trim() : ""; + + let body = raw; + body = body.replace(//gi, "\n"); + body = body.replace(//gi, "\n"); + body = body.replace(//gi, "\n"); + + // Preserve headings as section markers. + body = body.replace(/]*>([\s\S]*?)<\/h\1>/gi, (_m, lvl, text) => { + const t = decodeHtmlEntities(String(text)) + .replace(/<[^>]+>/g, " ") + .replace(/\s+/g, " ") + .trim(); + return `\n\n#${"#".repeat(Math.max(0, Number(lvl) - 1))} ${t}\n\n`; + }); + + // Newlines for common block separators. + body = body.replace(//gi, "\n"); + body = body.replace(/<\/(p|div|section|article|header|footer|li|ul|ol|table|tr)>/gi, "\n"); + body = body.replace(/<(p|div|section|article|header|footer|li|ul|ol|table|tr)[^>]*>/gi, "\n"); + + // Strip tags and decode entities. + body = decodeHtmlEntities(body.replace(/<[^>]+>/g, " ")); + body = body.replace(/\r\n/g, "\n"); + body = body.replace(/[ \t]+\n/g, "\n"); + body = body.replace(/\n{3,}/g, "\n\n").trim(); + + if (!body) { + warnings.push("empty_html_body"); + } + + const blocks = body + .split(/\n{2,}/g) + .map((b) => b.trim()) + .filter(Boolean); + const segments: ExtractionSegmentV1[] = []; + let currentLabel: string | undefined = title || undefined; + + for (const block of blocks) { + if (block.startsWith("#")) { + currentLabel = block.replace(/^#+\s*/, "").trim() || currentLabel; + continue; + } + const chunks = splitIntoChunks({ + text: block, + maxChars: params.chunkMaxChars, + overlapChars: params.chunkOverlapChars, + }); + for (let i = 0; i < chunks.length; i += 1) { + const c = chunks[i]; + if (!c) { + continue; + } + segments.push({ + text: c.text, + locator: { + kind: "section", + index: segments.length + 1, + total: 0, // set below + label: currentLabel, + startChar: c.startChar, + endChar: c.endChar, + }, + }); + } + } + for (let i = 0; i < segments.length; i += 1) { + const seg = segments[i]; + if (!seg) { + continue; + } + if (seg.locator) { + seg.locator.index = i + 1; + seg.locator.total = segments.length; + } + } + + return { + schema_version: 1, + docTypeGuess: "html", + languageGuess: "html", + segments, + warnings, + stats: { + bytes: params.bytes, + chars: raw.length, + segments: segments.length, + truncated: params.truncated, + }, + }; +} + +function guessDocType(mime: string | undefined, filename: string): string { + const m = (mime ?? "").toLowerCase().trim(); + const lower = filename.toLowerCase(); + if ( + m === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || + lower.endsWith(".docx") + ) { + return "docx"; + } + if ( + m === "application/vnd.openxmlformats-officedocument.presentationml.presentation" || + lower.endsWith(".pptx") + ) { + return "pptx"; + } + if (m === "application/pdf" || lower.endsWith(".pdf")) { + return "pdf"; + } + if ( + m === "text/html" || + m === "application/xhtml+xml" || + lower.endsWith(".html") || + lower.endsWith(".htm") + ) { + return "html"; + } + if (m.includes("markdown") || lower.endsWith(".md") || lower.endsWith(".markdown")) { + return "markdown"; + } + if (m.includes("json") || lower.endsWith(".json") || lower.endsWith(".jsonl")) { + return "json"; + } + if (m.includes("yaml") || lower.endsWith(".yaml") || lower.endsWith(".yml")) { + return "yaml"; + } + if (m.includes("xml") || lower.endsWith(".xml")) { + return "xml"; + } + if (m.startsWith("text/") || lower.endsWith(".txt")) { + return "text"; + } + return m ? `mime:${m}` : "unknown"; +} + +function guessLanguage(filename: string): string | undefined { + const lower = filename.toLowerCase(); + const ext = lower.split(".").pop() ?? ""; + const map: Record = { + ts: "typescript", + tsx: "typescript", + js: "javascript", + jsx: "javascript", + py: "python", + go: "go", + rs: "rust", + java: "java", + kt: "kotlin", + swift: "swift", + md: "markdown", + json: "json", + yaml: "yaml", + yml: "yaml", + toml: "toml", + xml: "xml", + html: "html", + htm: "html", + css: "css", + sh: "shell", + }; + return map[ext]; +} + +async function fetchJson(params: { + url: string; + method: "GET" | "POST"; + apiKey?: string; + body?: unknown; + timeoutMs: number; +}): Promise { + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), params.timeoutMs); + try { + const res = await fetch(params.url, { + method: params.method, + headers: { + ...(params.apiKey ? { "x-api-key": params.apiKey } : {}), + ...(params.body ? { "content-type": "application/json" } : {}), + }, + body: params.body ? JSON.stringify(params.body) : undefined, + signal: ctrl.signal, + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + throw new Error(`HTTP ${res.status} ${res.statusText}${text ? `: ${text}` : ""}`); + } + return (await res.json()) as T; + } finally { + clearTimeout(timer); + } +} + +function isOverloadLikeError(message: string): boolean { + const m = message.toLowerCase(); + return ( + m.includes("http 503") || + m.includes("queue_overloaded") || + m.includes("degraded_read_only") || + m.includes("namespace_overloaded") || + m.includes("rate limit") || + m.includes("http 429") + ); +} + +async function fetchBytes(params: { + url: string; + apiKey?: string; + timeoutMs: number; + maxBytes: number; +}): Promise { + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), params.timeoutMs); + try { + const headers: Record = {}; + if (params.apiKey) { + headers["x-api-key"] = params.apiKey; + } + const res = await fetch(params.url, { + method: "GET", + headers, + signal: ctrl.signal, + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + throw new Error(`HTTP ${res.status} ${res.statusText}${text ? `: ${text}` : ""}`); + } + const reader = res.body?.getReader(); + if (!reader) { + const buf = new Uint8Array(await res.arrayBuffer()); + return buf.length > params.maxBytes ? buf.slice(0, params.maxBytes) : buf; + } + const chunks: Uint8Array[] = []; + let total = 0; + while (true) { + const { value, done } = await reader.read(); + if (done) { + break; + } + if (!value) { + continue; + } + total += value.length; + if (total > params.maxBytes) { + chunks.push(value.slice(0, Math.max(0, params.maxBytes - (total - value.length)))); + break; + } + chunks.push(value); + } + const out = new Uint8Array(chunks.reduce((sum, c) => sum + c.length, 0)); + let offset = 0; + for (const c of chunks) { + out.set(c, offset); + offset += c.length; + } + return out; + } finally { + clearTimeout(timer); + } +} + +function splitIntoChunks(params: { + text: string; + maxChars: number; + overlapChars: number; +}): Array<{ text: string; truncated: boolean; startChar: number; endChar: number }> { + const maxChars = Math.max(200, params.maxChars); + const overlap = Math.max(0, Math.min(params.overlapChars, Math.max(0, maxChars - 50))); + const t = params.text; + if (!t.trim()) { + return []; + } + const chunks: Array<{ text: string; truncated: boolean; startChar: number; endChar: number }> = + []; + let i = 0; + while (i < t.length) { + const end = Math.min(t.length, i + maxChars); + const slice = t.slice(i, end); + chunks.push({ text: slice, truncated: end < t.length, startChar: i, endChar: end }); + if (end >= t.length) { + break; + } + i = Math.max(0, end - overlap); + } + return chunks; +} + +function extractTextLike(params: { + file: RustFsFileMeta; + text: string; + bytes: number; + truncated: boolean; + chunkMaxChars: number; + chunkOverlapChars: number; +}): ExtractionResultV1 { + const chunks = splitIntoChunks({ + text: params.text, + maxChars: params.chunkMaxChars, + overlapChars: params.chunkOverlapChars, + }); + const segments: ExtractionSegmentV1[] = chunks.map((c, idx) => ({ + text: c.text, + locator: { + kind: "chunk", + index: idx + 1, + total: chunks.length, + startChar: c.startChar, + endChar: c.endChar, + }, + })); + return { + schema_version: 1, + docTypeGuess: guessDocType(params.file.mime ?? undefined, params.file.filename), + languageGuess: guessLanguage(params.file.filename), + segments, + warnings: [], + stats: { + bytes: params.bytes, + chars: params.text.length, + segments: segments.length, + truncated: params.truncated, + }, + }; +} + +function extractCodeLike(params: { + file: RustFsFileMeta; + code: string; + bytes: number; + truncated: boolean; + chunkMaxChars: number; + chunkOverlapChars: number; +}): ExtractionResultV1 { + const warnings: string[] = []; + const lang = guessLanguage(params.file.filename); + const code = params.code.replace(/\r\n/g, "\n"); + const lines = code.split("\n"); + + const boundaryPatterns: Array<{ lang?: string; re: RegExp }> = [ + { lang: "python", re: /^\s*(def|class)\s+[A-Za-z0-9_]+\s*\(/ }, + { lang: "go", re: /^\s*func\s+(\([^)]+\)\s*)?[A-Za-z0-9_]+\s*\(/ }, + { lang: "rust", re: /^\s*(pub\s+)?(async\s+)?fn\s+[A-Za-z0-9_]+\s*\(/ }, + { + lang: "java", + re: /^\s*(public|private|protected)?\s*(static\s+)?(class|interface)\s+[A-Za-z0-9_]+/, + }, + { + lang: "typescript", + re: /^\s*(export\s+)?(async\s+)?(function|class|interface|type)\s+[A-Za-z0-9_]+/, + }, + { + lang: "javascript", + re: /^\s*(export\s+)?(async\s+)?(function|class)\s+[A-Za-z0-9_]+/, + }, + ]; + + const chosen = boundaryPatterns.filter((p) => !p.lang || p.lang === lang); + const isBoundary = (line: string): boolean => chosen.some((p) => p.re.test(line)); + + const maxChars = Math.max(500, params.chunkMaxChars); + const overlap = Math.max(0, Math.min(params.chunkOverlapChars, Math.max(0, maxChars - 100))); + + const segments: ExtractionSegmentV1[] = []; + let buf: string[] = []; + let bufStartLine = 0; + + const flush = (endLineExclusive: number) => { + const text = buf.join("\n").trimEnd(); + if (!text.trim()) { + buf = []; + bufStartLine = endLineExclusive; + return; + } + // If a block is still too large, fall back to char-based split. + if (text.length > maxChars) { + const chunks = splitIntoChunks({ text, maxChars, overlapChars: overlap }); + for (let i = 0; i < chunks.length; i += 1) { + const c = chunks[i]; + if (!c) { + continue; + } + segments.push({ + text: c.text, + locator: { + kind: "section", + index: 0, + total: 0, + label: `lines ${bufStartLine + 1}-${endLineExclusive}`, + }, + }); + } + } else { + segments.push({ + text, + locator: { + kind: "section", + index: 0, + total: 0, + label: `lines ${bufStartLine + 1}-${endLineExclusive}`, + }, + }); + } + buf = []; + bufStartLine = endLineExclusive; + }; + + for (let i = 0; i < lines.length; i += 1) { + const line = lines[i] ?? ""; + if (buf.length === 0) { + bufStartLine = i; + } + // Start a new segment at structural boundaries (best-effort). + if (buf.length > 0 && isBoundary(line) && buf.join("\n").length >= Math.floor(maxChars * 0.6)) { + flush(i); + } + buf.push(line); + if (buf.join("\n").length >= maxChars) { + flush(i + 1); + } + } + if (buf.length > 0) { + flush(lines.length); + } + + if (segments.length === 0 && code.trim()) { + warnings.push("code_chunking_failed_fallback_to_textlike"); + return extractTextLike({ + file: params.file, + text: code, + bytes: params.bytes, + truncated: params.truncated, + chunkMaxChars: params.chunkMaxChars, + chunkOverlapChars: params.chunkOverlapChars, + }); + } + + for (let i = 0; i < segments.length; i += 1) { + const seg = segments[i]; + if (!seg?.locator) { + continue; + } + seg.locator.index = i + 1; + seg.locator.total = segments.length; + } + + return { + schema_version: 1, + docTypeGuess: "code", + languageGuess: lang, + segments, + warnings, + stats: { + bytes: params.bytes, + chars: code.length, + segments: segments.length, + truncated: params.truncated, + }, + }; +} + +async function extractPdfLike(params: { + file: RustFsFileMeta; + bytes: Uint8Array; + truncated: boolean; + chunkMaxChars: number; + chunkOverlapChars: number; +}): Promise { + const warnings: string[] = []; + const buf = Buffer.from(params.bytes); + const maxChars = Math.max(500, params.chunkMaxChars); + const overlap = Math.max(0, Math.min(params.chunkOverlapChars, Math.max(0, maxChars - 100))); + try { + const parser = new PDFParse({ data: buf }); + const textRes = await parser.getText(); + await parser.destroy(); + + const pages = (textRes.pages ?? []).map((p) => p.text ?? ""); + if (pages.every((p) => !p.trim())) { + warnings.push("empty_pdf_text"); + } + + const segments: ExtractionSegmentV1[] = []; + const totalPages = pages.length || 1; + for (let pi = 0; pi < pages.length; pi += 1) { + const pageText = pages[pi] ?? ""; + if (!pageText.trim()) { + continue; + } + const chunks = splitIntoChunks({ text: pageText, maxChars, overlapChars: overlap }); + for (let ci = 0; ci < chunks.length; ci += 1) { + const c = chunks[ci]; + if (!c) { + continue; + } + segments.push({ + text: c.text, + locator: { + kind: "page", + index: pi + 1, + total: totalPages, + label: `page ${pi + 1}`, + startChar: c.startChar, + endChar: c.endChar, + }, + }); + } + } + + return { + schema_version: 1, + docTypeGuess: "pdf", + languageGuess: undefined, + segments, + warnings, + stats: { + bytes: params.bytes.length, + chars: pages.reduce((sum, p) => sum + p.length, 0), + segments: segments.length, + truncated: params.truncated, + }, + }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + warnings.push(`pdf_parse_failed:${message}`); + return { + schema_version: 1, + docTypeGuess: "pdf", + languageGuess: undefined, + segments: [], + warnings, + stats: { + bytes: params.bytes.length, + chars: 0, + segments: 0, + truncated: params.truncated, + }, + }; + } +} + +async function extractDocxLike(params: { + file: RustFsFileMeta; + bytes: Uint8Array; + truncated: boolean; + chunkMaxChars: number; + chunkOverlapChars: number; +}): Promise { + const warnings: string[] = []; + const buf = Buffer.from(params.bytes); + let value = ""; + try { + const out = await mammoth.extractRawText({ buffer: buf }); + value = out.value ?? ""; + const msgs = out.messages ?? []; + for (const m of msgs) { + if (m?.type && m?.message) { + warnings.push(`docx:${m.type}:${m.message}`); + } + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + warnings.push(`docx_extract_failed:${message}`); + return { + schema_version: 1, + docTypeGuess: "docx", + languageGuess: undefined, + segments: [], + warnings, + stats: { + bytes: params.bytes.length, + chars: 0, + segments: 0, + truncated: params.truncated, + }, + }; + } + + const text = value.replace(/\r\n/g, "\n").trim(); + const blocks = text + .split(/\n{2,}/g) + .map((b) => b.trim()) + .filter(Boolean); + const segments: ExtractionSegmentV1[] = []; + const maxChars = Math.max(500, params.chunkMaxChars); + const overlap = Math.max(0, Math.min(params.chunkOverlapChars, Math.max(0, maxChars - 100))); + for (let bi = 0; bi < blocks.length; bi += 1) { + const block = blocks[bi] ?? ""; + if (!block.trim()) { + continue; + } + const chunks = splitIntoChunks({ text: block, maxChars, overlapChars: overlap }); + for (let ci = 0; ci < chunks.length; ci += 1) { + const c = chunks[ci]; + if (!c) { + continue; + } + segments.push({ + text: c.text, + locator: { + kind: "section", + index: segments.length + 1, + total: 0, + label: `paragraph ${bi + 1}`, + startChar: c.startChar, + endChar: c.endChar, + }, + }); + } + } + for (let i = 0; i < segments.length; i += 1) { + const seg = segments[i]; + if (!seg?.locator) { + continue; + } + seg.locator.index = i + 1; + seg.locator.total = segments.length; + } + + if (segments.length === 0 && text) { + warnings.push("empty_docx_text"); + } + + return { + schema_version: 1, + docTypeGuess: "docx", + languageGuess: undefined, + segments, + warnings, + stats: { + bytes: params.bytes.length, + chars: text.length, + segments: segments.length, + truncated: params.truncated, + }, + }; +} + +async function extractPptxLike(params: { + file: RustFsFileMeta; + bytes: Uint8Array; + truncated: boolean; + chunkMaxChars: number; + chunkOverlapChars: number; +}): Promise { + const warnings: string[] = []; + const buf = Buffer.from(params.bytes); + const zip = await JSZip.loadAsync(buf).catch((err) => { + const message = err instanceof Error ? err.message : String(err); + warnings.push(`pptx_zip_failed:${message}`); + return null; + }); + if (!zip) { + return { + schema_version: 1, + docTypeGuess: "pptx", + languageGuess: undefined, + segments: [], + warnings, + stats: { + bytes: params.bytes.length, + chars: 0, + segments: 0, + truncated: params.truncated, + }, + }; + } + + const slideNames = Object.keys(zip.files) + .filter((n) => /^ppt\/slides\/slide\d+\.xml$/.test(n)) + .toSorted((a, b) => { + const ai = Number(a.match(/slide(\d+)\.xml$/)?.[1] ?? 0); + const bi = Number(b.match(/slide(\d+)\.xml$/)?.[1] ?? 0); + return ai - bi; + }); + const totalSlides = slideNames.length || 1; + + const segments: ExtractionSegmentV1[] = []; + const maxChars = Math.max(500, params.chunkMaxChars); + const overlap = Math.max(0, Math.min(params.chunkOverlapChars, Math.max(0, maxChars - 100))); + for (let si = 0; si < slideNames.length; si += 1) { + const name = slideNames[si]; + if (!name) { + continue; + } + const xml = await zip + .file(name) + ?.async("string") + .catch(() => null); + if (!xml) { + warnings.push(`pptx_slide_read_failed:${name}`); + continue; + } + const runs: string[] = []; + const re = /]*>([\s\S]*?)<\/a:t>/g; + let m: RegExpExecArray | null; + while ((m = re.exec(xml))) { + const raw = decodeHtmlEntities(m[1] ?? ""); + const t = raw.replace(/\s+/g, " ").trim(); + if (t) { + runs.push(t); + } + } + const slideText = runs.join(" ").trim(); + if (!slideText) { + continue; + } + const chunks = splitIntoChunks({ text: slideText, maxChars, overlapChars: overlap }); + for (let ci = 0; ci < chunks.length; ci += 1) { + const c = chunks[ci]; + if (!c) { + continue; + } + segments.push({ + text: c.text, + locator: { + kind: "slide", + index: si + 1, + total: totalSlides, + label: `slide ${si + 1}`, + startChar: c.startChar, + endChar: c.endChar, + }, + }); + } + } + + if (segments.length === 0) { + warnings.push("empty_pptx_text"); + } + + return { + schema_version: 1, + docTypeGuess: "pptx", + languageGuess: undefined, + segments, + warnings, + stats: { + bytes: params.bytes.length, + chars: segments.reduce((sum, s) => sum + s.text.length, 0), + segments: segments.length, + truncated: params.truncated, + }, + }; +} + +function buildDeepMemoryMessages(params: { + file: RustFsFileMeta; + extraction: ExtractionResultV1; +}): unknown[] { + const header = [ + "FILE_CONTEXT", + `file_id: ${params.file.file_id}`, + `tenant_id: ${params.file.tenant_id}`, + `session_id: ${params.file.session_id ?? ""}`, + `filename: ${params.file.filename}`, + `mime: ${params.file.mime ?? ""}`, + `sha256: ${params.file.sha256}`, + `size: ${params.file.size}`, + `created_at_ms: ${params.file.created_at_ms}`, + `doc_type_guess: ${params.extraction.docTypeGuess}`, + `language_guess: ${params.extraction.languageGuess ?? ""}`, + `truncated: ${params.extraction.stats.truncated}`, + ].join("\n"); + + const total = params.extraction.segments.length; + return params.extraction.segments.map((seg, idx) => { + const loc = seg.locator; + const locLines = loc + ? [ + `segment: ${idx + 1}/${total}`, + `locator_kind: ${loc.kind}`, + `locator_index: ${loc.index}/${loc.total}`, + `locator_label: ${loc.label ?? ""}`, + `start_char: ${loc.startChar ?? ""}`, + `end_char: ${loc.endChar ?? ""}`, + ].join("\n") + : `segment: ${idx + 1}/${total}`; + const content = `${header}\n${locLines}\n\nCONTENT_EXCERPT\n${seg.text}`; + return { role: "user", content }; + }); +} + +const WorkerStateSchema = z.object({ + tombstoneSinceMs: z.coerce.number().int().nonnegative().default(0), +}); + +type WorkerState = z.infer; + +async function loadWorkerState( + path: string | undefined, + fallback: WorkerState, +): Promise { + const p = path?.trim(); + if (!p) { + return fallback; + } + try { + const raw = await readFile(p, "utf-8"); + return WorkerStateSchema.parse(JSON.parse(raw)); + } catch { + return fallback; + } +} + +async function saveWorkerState(path: string | undefined, state: WorkerState): Promise { + const p = path?.trim(); + if (!p) { + return; + } + await mkdir(dirname(p), { recursive: true }); + await writeFile(p, JSON.stringify(state, null, 2), { encoding: "utf-8" }); +} + +async function main(): Promise { + const env = EnvSchema.parse(process.env); + const log = pino({ name: "rustfs-worker" }); + + const rustfsBaseUrl = normalizeBaseUrl(env.RUSTFS_BASE_URL); + const deepBaseUrl = normalizeBaseUrl(env.DEEP_MEMORY_BASE_URL); + + const loadedState = await loadWorkerState(env.WORKER_STATE_PATH, { + tombstoneSinceMs: env.RUSTFS_TOMBSTONE_SINCE_MS, + }); + let tombstoneSinceMs = Math.max(env.RUSTFS_TOMBSTONE_SINCE_MS, loadedState.tombstoneSinceMs); + let lastTombstonePollAt = 0; + + log.info( + { + rustfsBaseUrl, + deepBaseUrl, + tenantHint: env.RUSTFS_TENANT_ID ?? undefined, + pollIntervalMs: env.RUSTFS_POLL_INTERVAL_MS, + leaseMs: env.RUSTFS_LEASE_MS, + pendingLimit: env.RUSTFS_PENDING_LIMIT, + maxDownloadBytes: env.RUSTFS_MAX_DOWNLOAD_BYTES, + tombstonePollIntervalMs: env.RUSTFS_TOMBSTONE_POLL_INTERVAL_MS, + tombstoneLimit: env.RUSTFS_TOMBSTONE_LIMIT, + tombstoneSinceMs, + deepNamespace: env.DEEP_MEMORY_NAMESPACE ?? undefined, + deepAsync: env.DEEP_MEMORY_ASYNC, + }, + "worker starting", + ); + + async function pollTombstonesIfDue(): Promise { + const now = Date.now(); + if (now - lastTombstonePollAt < env.RUSTFS_TOMBSTONE_POLL_INTERVAL_MS) { + return; + } + lastTombstonePollAt = now; + const tombUrl = new URL(`${rustfsBaseUrl}/v1/files/tombstoned`); + if (env.RUSTFS_TENANT_ID?.trim()) { + tombUrl.searchParams.set("tenant_id", env.RUSTFS_TENANT_ID.trim()); + } + tombUrl.searchParams.set("since_ms", String(tombstoneSinceMs)); + tombUrl.searchParams.set("limit", String(env.RUSTFS_TOMBSTONE_LIMIT)); + + const tomb = await fetchJson<{ + ok: boolean; + items: Array<{ file_id: string; deleted_at_ms: number }>; + }>({ + url: tombUrl.toString(), + method: "GET", + apiKey: env.RUSTFS_API_KEY, + timeoutMs: env.DEEP_MEMORY_TIMEOUT_MS, + }); + + const tItems = tomb.items ?? []; + for (const item of tItems) { + const sessionId = `rustfs:file:${item.file_id}`; + let res: unknown; + try { + res = await fetchJson({ + url: `${deepBaseUrl}/forget`, + method: "POST", + apiKey: env.DEEP_MEMORY_API_KEY, + timeoutMs: env.DEEP_MEMORY_TIMEOUT_MS, + body: { + namespace: env.DEEP_MEMORY_NAMESPACE?.trim() || undefined, + session_id: sessionId, + async: true, + }, + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (isOverloadLikeError(message)) { + log.warn({ err: message }, "deep-memory overloaded during forget; backing off"); + await sleep(env.DEEP_MEMORY_BACKOFF_MS); + // Do not advance tombstoneSinceMs so we retry later. + throw err; + } + throw err; + } + log.info( + { file_id: item.file_id, sessionId, deleted_at_ms: item.deleted_at_ms, res }, + "forgot tombstoned file", + ); + tombstoneSinceMs = Math.max(tombstoneSinceMs, item.deleted_at_ms); + await saveWorkerState(env.WORKER_STATE_PATH, { tombstoneSinceMs }).catch(() => {}); + } + } + + let overloadHits = 0; + let lastEmptyLogAt = 0; + + while (true) { + try { + const pending = await fetchJson<{ + ok: boolean; + items: RustFsFileMeta[]; + claimed_at_ms?: number; + }>({ + url: `${rustfsBaseUrl}/v1/files/claim_extract`, + method: "POST", + apiKey: env.RUSTFS_API_KEY, + timeoutMs: env.DEEP_MEMORY_TIMEOUT_MS, + body: { + tenant_id: env.RUSTFS_TENANT_ID?.trim() || undefined, + limit: env.RUSTFS_PENDING_LIMIT, + lease_ms: env.RUSTFS_LEASE_MS, + }, + }); + + const items = pending.items ?? []; + try { + await pollTombstonesIfDue(); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + log.warn({ err: message }, "tombstone forget poll failed"); + } + + if (items.length === 0) { + const now = Date.now(); + if (now - lastEmptyLogAt > 60_000) { + lastEmptyLogAt = now; + log.debug( + { + pollIntervalMs: env.RUSTFS_POLL_INTERVAL_MS, + tombstoneSinceMs, + overloadHits, + }, + "no pending files to extract", + ); + } + await sleep(env.RUSTFS_POLL_INTERVAL_MS); + continue; + } + + for (const file of items) { + const fileLog = log.child({ file_id: file.file_id, filename: file.filename }); + try { + const t0 = Date.now(); + const canText = isSupportedTextLike(file.mime ?? undefined, file.filename); + const canHtml = isHtmlLike(file.mime ?? undefined, file.filename); + const canCode = isCodeLike(file.mime ?? undefined, file.filename); + const canPdf = isPdfLike(file.mime ?? undefined, file.filename); + const canDocx = isDocxLike(file.mime ?? undefined, file.filename); + const canPptx = isPptxLike(file.mime ?? undefined, file.filename); + if (!canText && !canHtml && !canCode && !canPdf && !canDocx && !canPptx) { + await fetchJson({ + url: `${rustfsBaseUrl}/v1/files/${encodeURIComponent(file.file_id)}/extract_status`, + method: "POST", + apiKey: env.RUSTFS_API_KEY, + timeoutMs: env.DEEP_MEMORY_TIMEOUT_MS, + body: { status: "skipped", error: `unsupported_mime:${file.mime ?? ""}` }, + }); + fileLog.info({ mime: file.mime ?? undefined }, "skipped unsupported mime"); + continue; + } + + const tDownload0 = Date.now(); + const bytes = await fetchBytes({ + url: `${rustfsBaseUrl}/v1/files/${encodeURIComponent(file.file_id)}`, + apiKey: env.RUSTFS_API_KEY, + timeoutMs: env.DEEP_MEMORY_TIMEOUT_MS, + maxBytes: env.RUSTFS_MAX_DOWNLOAD_BYTES, + }); + const downloadMs = Date.now() - tDownload0; + const truncated = bytes.length >= env.RUSTFS_MAX_DOWNLOAD_BYTES; + const text = + canText || canHtml || canCode + ? new TextDecoder("utf-8", { fatal: false }).decode(bytes) + : ""; + + const sessionId = `rustfs:file:${file.file_id}`; + const tExtract0 = Date.now(); + const extraction = canPdf + ? await extractPdfLike({ + file, + bytes, + truncated, + chunkMaxChars: env.DEEP_MEMORY_CHUNK_MAX_CHARS, + chunkOverlapChars: env.DEEP_MEMORY_CHUNK_OVERLAP_CHARS, + }) + : canDocx + ? await extractDocxLike({ + file, + bytes, + truncated, + chunkMaxChars: env.DEEP_MEMORY_CHUNK_MAX_CHARS, + chunkOverlapChars: env.DEEP_MEMORY_CHUNK_OVERLAP_CHARS, + }) + : canPptx + ? await extractPptxLike({ + file, + bytes, + truncated, + chunkMaxChars: env.DEEP_MEMORY_CHUNK_MAX_CHARS, + chunkOverlapChars: env.DEEP_MEMORY_CHUNK_OVERLAP_CHARS, + }) + : canHtml + ? extractHtmlLike({ + file, + html: text, + bytes: bytes.length, + truncated, + chunkMaxChars: env.DEEP_MEMORY_CHUNK_MAX_CHARS, + chunkOverlapChars: env.DEEP_MEMORY_CHUNK_OVERLAP_CHARS, + }) + : canCode + ? extractCodeLike({ + file, + code: text, + bytes: bytes.length, + truncated, + chunkMaxChars: env.DEEP_MEMORY_CHUNK_MAX_CHARS, + chunkOverlapChars: env.DEEP_MEMORY_CHUNK_OVERLAP_CHARS, + }) + : extractTextLike({ + file, + text, + bytes: bytes.length, + truncated, + chunkMaxChars: env.DEEP_MEMORY_CHUNK_MAX_CHARS, + chunkOverlapChars: env.DEEP_MEMORY_CHUNK_OVERLAP_CHARS, + }); + const extractMs = Date.now() - tExtract0; + if (extraction.segments.length === 0) { + await fetchJson({ + url: `${rustfsBaseUrl}/v1/files/${encodeURIComponent(file.file_id)}/extract_status`, + method: "POST", + apiKey: env.RUSTFS_API_KEY, + timeoutMs: env.DEEP_MEMORY_TIMEOUT_MS, + body: { status: "skipped", error: "empty_text" }, + }); + fileLog.info({ sessionId }, "skipped empty text"); + continue; + } + const tMsg0 = Date.now(); + const messages = buildDeepMemoryMessages({ file, extraction }); + const messageBuildMs = Date.now() - tMsg0; + let updateRes: unknown; + const tDeep0 = Date.now(); + try { + const wantMemoryIds = env.DEEP_MEMORY_RETURN_MEMORY_IDS && !env.DEEP_MEMORY_ASYNC; + updateRes = await fetchJson({ + url: `${deepBaseUrl}/update_memory_index`, + method: "POST", + apiKey: env.DEEP_MEMORY_API_KEY, + timeoutMs: env.DEEP_MEMORY_TIMEOUT_MS, + body: { + namespace: env.DEEP_MEMORY_NAMESPACE?.trim() || undefined, + session_id: sessionId, + messages, + async: env.DEEP_MEMORY_ASYNC, + return_memory_ids: wantMemoryIds ? true : undefined, + max_memory_ids: wantMemoryIds ? env.DEEP_MEMORY_MAX_RETURN_MEMORY_IDS : undefined, + }, + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (isOverloadLikeError(message)) { + overloadHits += 1; + const annotations = buildAnnotationsV1({ + file, + existingAnnotations: file.annotations, + deepMemory: { + baseUrl: deepBaseUrl, + namespace: env.DEEP_MEMORY_NAMESPACE?.trim() || undefined, + sessionId, + error: message, + overloaded: true, + }, + extraction, + semantics: undefined, + }); + await fetchJson({ + url: `${rustfsBaseUrl}/v1/files/${encodeURIComponent(file.file_id)}/annotations`, + method: "POST", + apiKey: env.RUSTFS_API_KEY, + timeoutMs: env.DEEP_MEMORY_TIMEOUT_MS, + body: { annotations, source: "rustfs-worker" }, + }).catch(() => {}); + fileLog.warn({ err: message }, "deep-memory overloaded; backing off"); + await sleep(env.DEEP_MEMORY_BACKOFF_MS); + } + throw err; + } + const deepMemoryMs = Date.now() - tDeep0; + + const tAnno0 = Date.now(); + const memIds = extractMemoryIdsFromUpdateResponse(updateRes); + + let semantics: { topics?: string[]; entities?: string[]; summary?: string } | undefined; + if (env.DEEP_MEMORY_INSPECT_ENABLED) { + try { + const inspectRes = await fetchJson({ + url: `${deepBaseUrl}/session/inspect`, + method: "POST", + apiKey: env.DEEP_MEMORY_API_KEY, + timeoutMs: env.DEEP_MEMORY_TIMEOUT_MS, + body: { + namespace: env.DEEP_MEMORY_NAMESPACE?.trim() || undefined, + session_id: sessionId, + limit: env.DEEP_MEMORY_INSPECT_LIMIT, + include_content: env.DEEP_MEMORY_INSPECT_INCLUDE_CONTENT, + }, + }); + semantics = extractSessionSemanticsFromInspectResponse(inspectRes); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (isOverloadLikeError(message)) { + overloadHits += 1; + fileLog.warn({ err: message }, "deep-memory overloaded during session inspect"); + } else { + fileLog.warn({ err: message }, "session inspect failed"); + } + } + } + const annotations = buildAnnotationsV1({ + file, + existingAnnotations: file.annotations, + deepMemory: { + baseUrl: deepBaseUrl, + namespace: env.DEEP_MEMORY_NAMESPACE?.trim() || undefined, + sessionId, + updateResponse: updateRes, + memoryIds: memIds.ids, + memoryIdsTruncated: memIds.truncated, + }, + extraction, + semantics, + }); + + await fetchJson({ + url: `${rustfsBaseUrl}/v1/files/${encodeURIComponent(file.file_id)}/annotations`, + method: "POST", + apiKey: env.RUSTFS_API_KEY, + timeoutMs: env.DEEP_MEMORY_TIMEOUT_MS, + body: { annotations, source: "rustfs-worker" }, + }); + await fetchJson({ + url: `${rustfsBaseUrl}/v1/files/${encodeURIComponent(file.file_id)}/extract_status`, + method: "POST", + apiKey: env.RUSTFS_API_KEY, + timeoutMs: env.DEEP_MEMORY_TIMEOUT_MS, + body: { status: "indexed" }, + }); + const rustfsWriteMs = Date.now() - tAnno0; + const totalMs = Date.now() - t0; + + fileLog.info( + { + sessionId, + docType: extraction.docTypeGuess, + segments: extraction.segments.length, + bytes: extraction.stats.bytes, + truncated: extraction.stats.truncated, + timings: { + downloadMs, + extractMs, + messageBuildMs, + deepMemoryMs, + rustfsWriteMs, + totalMs, + }, + }, + "indexed", + ); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + await fetchJson({ + url: `${rustfsBaseUrl}/v1/files/${encodeURIComponent(file.file_id)}/extract_status`, + method: "POST", + apiKey: env.RUSTFS_API_KEY, + timeoutMs: env.DEEP_MEMORY_TIMEOUT_MS, + body: { status: "failed", error: message }, + }).catch(() => {}); + fileLog.warn({ err: message }, "failed"); + } + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + // Best-effort global loop resilience. + // eslint-disable-next-line no-console + console.error(`[rustfs-worker] loop error: ${message}`); + await sleep(env.RUSTFS_POLL_INTERVAL_MS); + } + } +} + +await main(); diff --git a/packages/rustfs-worker/src/vendor.d.ts b/packages/rustfs-worker/src/vendor.d.ts new file mode 100644 index 0000000000000..dea04cce1c2d5 --- /dev/null +++ b/packages/rustfs-worker/src/vendor.d.ts @@ -0,0 +1,7 @@ +declare module "mammoth" { + export type MammothMessage = { type: string; message: string }; + export type ExtractRawTextResult = { value: string; messages?: MammothMessage[] }; + export function extractRawText(input: { buffer: Buffer }): Promise; + const mammoth: { extractRawText: typeof extractRawText }; + export default mammoth; +} diff --git a/packages/rustfs-worker/tsconfig.json b/packages/rustfs-worker/tsconfig.json new file mode 100644 index 0000000000000..00ce269fbe17d --- /dev/null +++ b/packages/rustfs-worker/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "noEmit": false, + "allowImportingTsExtensions": false + }, + "include": ["src/**/*.ts", "src/**/*.d.ts"] +} diff --git a/packages/rustfs/Cargo.lock b/packages/rustfs/Cargo.lock new file mode 100644 index 0000000000000..3e6ac4f2024a9 --- /dev/null +++ b/packages/rustfs/Cargo.lock @@ -0,0 +1,2045 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "age" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf640be7658959746f1f0f2faab798f6098a9436a8e18e148d18bc9875e13c4b" +dependencies = [ + "age-core", + "base64 0.21.7", + "bech32", + "chacha20poly1305", + "cookie-factory", + "hmac", + "i18n-embed", + "i18n-embed-fl", + "lazy_static", + "nom", + "pin-project", + "rand", + "rust-embed", + "scrypt", + "sha2", + "subtle", + "x25519-dalek", + "zeroize", +] + +[[package]] +name = "age-core" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2bf6a89c984ca9d850913ece2da39e1d200563b0a94b002b253beee4c5acf99" +dependencies = [ + "base64 0.21.7", + "chacha20poly1305", + "cookie-factory", + "hkdf", + "io_tee", + "nom", + "rand", + "secrecy", + "sha2", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "arc-swap" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ded5f9a03ac8f24d1b8a25101ee812cd32cdc8c50a4c50237de2c4915850e73" +dependencies = [ + "rustversion", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "axum" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "multer", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "basic-toml" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba62675e8242a4c4e806d12f11d136e626e6c8361d6b829310732241652a178a" +dependencies = [ + "serde", +] + +[[package]] +name = "bech32" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d86b93f97252c47b41663388e6d155714a9d0c398b99f1005cbc5f978b29f445" + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cc" +version = "1.2.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chacha20" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "chacha20poly1305" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +dependencies = [ + "aead", + "chacha20", + "cipher", + "poly1305", + "zeroize", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", + "zeroize", +] + +[[package]] +name = "cookie-factory" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" +dependencies = [ + "futures", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "find-crate" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a98bbaacea1c0eb6a0876280051b892eb73594fd90cf3b20e9c817029c57d2" +dependencies = [ + "toml", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fluent" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb74634707bebd0ce645a981148e8fb8c7bccd4c33c652aeffd28bf2f96d555a" +dependencies = [ + "fluent-bundle", + "unic-langid", +] + +[[package]] +name = "fluent-bundle" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe0a21ee80050c678013f82edf4b705fe2f26f1f9877593d13198612503f493" +dependencies = [ + "fluent-langneg", + "fluent-syntax", + "intl-memoizer", + "intl_pluralrules", + "rustc-hash 1.1.0", + "self_cell 0.10.3", + "smallvec", + "unic-langid", +] + +[[package]] +name = "fluent-langneg" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eebbe59450baee8282d71676f3bfed5689aeab00b27545e83e5f14b1195e8b0" +dependencies = [ + "unic-langid", +] + +[[package]] +name = "fluent-syntax" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a530c4694a6a8d528794ee9bbd8ba0122e779629ac908d15ad5a7ae7763a33d" +dependencies = [ + "thiserror 1.0.69", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "i18n-config" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e06b90c8a0d252e203c94344b21e35a30f3a3a85dc7db5af8f8df9f3e0c63ef" +dependencies = [ + "basic-toml", + "log", + "serde", + "serde_derive", + "thiserror 1.0.69", + "unic-langid", +] + +[[package]] +name = "i18n-embed" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "669ffc2c93f97e6ddf06ddbe999fcd6782e3342978bb85f7d3c087c7978404c4" +dependencies = [ + "arc-swap", + "fluent", + "fluent-langneg", + "fluent-syntax", + "i18n-embed-impl", + "intl-memoizer", + "log", + "parking_lot", + "rust-embed", + "thiserror 1.0.69", + "unic-langid", + "walkdir", +] + +[[package]] +name = "i18n-embed-fl" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04b2969d0b3fc6143776c535184c19722032b43e6a642d710fa3f88faec53c2d" +dependencies = [ + "find-crate", + "fluent", + "fluent-syntax", + "i18n-config", + "i18n-embed", + "proc-macro-error2", + "proc-macro2", + "quote", + "strsim", + "syn", + "unic-langid", +] + +[[package]] +name = "i18n-embed-impl" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f2cc0e0523d1fe6fc2c6f66e5038624ea8091b3e7748b5e8e0c84b1698db6c2" +dependencies = [ + "find-crate", + "i18n-config", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "intl-memoizer" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "310da2e345f5eb861e7a07ee182262e94975051db9e4223e909ba90f392f163f" +dependencies = [ + "type-map", + "unic-langid", +] + +[[package]] +name = "intl_pluralrules" +version = "7.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078ea7b7c29a2b4df841a7f6ac8775ff6074020c6776d48491ce2268e068f972" +dependencies = [ + "unic-langid", +] + +[[package]] +name = "io_tee" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b3f7cef34251886990511df1c61443aa928499d598a9473929ab5a90a527304" + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "js-sys" +version = "0.3.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.180" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" + +[[package]] +name = "libsqlite3-sys" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91632f3b4fb6bd1d72aa3d78f41ffecfcf2b1a6648d8c241dbe7dbfaf4875e15" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http", + "httparse", + "memchr", + "mime", + "spin", + "version_check", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "poly1305" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" +dependencies = [ + "cpufeatures", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" + +[[package]] +name = "rusqlite" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3de23c3319433716cf134eed225fe9986bc24f63bed9be9f20c329029e672dc7" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "rust-embed" +version = "8.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04113cb9355a377d83f06ef1f0a45b8ab8cd7d8b1288160717d66df5c7988d27" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0902e4c7c8e997159ab384e6d0fc91c221375f6894346ae107f47dd0f3ccaa" +dependencies = [ + "proc-macro2", + "quote", + "rust-embed-utils", + "syn", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bcdef0be6fe7f6fa333b1073c949729274b05f123a0ad7efcb8efd878e5c3b1" +dependencies = [ + "sha2", + "walkdir", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustfs" +version = "0.1.0" +dependencies = [ + "age", + "anyhow", + "axum", + "base64 0.22.1", + "bytes", + "hex", + "hmac", + "mime", + "rusqlite", + "secrecy", + "serde", + "serde_json", + "sha2", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tokio-util", + "tower-http", + "tracing", + "tracing-subscriber", + "uuid", +] + +[[package]] +name = "rustix" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" + +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2", +] + +[[package]] +name = "secrecy" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a" +dependencies = [ + "zeroize", +] + +[[package]] +name = "self_cell" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14e4d63b804dc0c7ec4a1e52bcb63f02c7ac94476755aa579edac21e01f915d" +dependencies = [ + "self_cell 1.2.2", +] + +[[package]] +name = "self_cell" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b12e76d157a900eb52e81bc6e9f3069344290341720e9178cde2407113ac8d89" + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "tempfile" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "serde_core", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags", + "bytes", + "http", + "http-body", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "type-map" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb30dbbd9036155e74adad6812e9898d03ec374946234fbcebd5dfc7b9187b90" +dependencies = [ + "rustc-hash 2.1.1", +] + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unic-langid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ba52c9b05311f4f6e62d5d9d46f094bd6e84cb8df7b3ef952748d752a7d05" +dependencies = [ + "unic-langid-impl", +] + +[[package]] +name = "unic-langid-impl" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce1bf08044d4b7a94028c93786f8566047edc11110595914de93362559bc658" +dependencies = [ + "serde", + "tinystr", +] + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + +[[package]] +name = "uuid" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee48d38b119b0cd71fe4141b30f5ba9c7c5d9f4e7a3a8b4a674e4b6ef789976f" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" + +[[package]] +name = "x25519-dalek" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" +dependencies = [ + "curve25519-dalek", + "rand_core", + "serde", + "zeroize", +] + +[[package]] +name = "zerocopy" +version = "0.8.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57cf3aa6855b23711ee9852dfc97dfaa51c45feaba5b645d0c777414d494a961" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a616990af1a287837c4fe6596ad77ef57948f787e46ce28e166facc0cc1cb75" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "serde", + "zerofrom", +] + +[[package]] +name = "zmij" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff05f8caa9038894637571ae6b9e29466c1f4f829d26c9b28f869a29cbe3445" diff --git a/packages/rustfs/Cargo.toml b/packages/rustfs/Cargo.toml new file mode 100644 index 0000000000000..eb7fef23da1fa --- /dev/null +++ b/packages/rustfs/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "rustfs" +version = "0.1.0" +edition = "2021" +license = "MIT" + +[dependencies] +anyhow = "1.0.99" +axum = { version = "0.8.1", features = ["multipart"] } +bytes = "1.10.1" +hex = "0.4.3" +mime = "0.3.17" +rusqlite = { version = "0.36.0", features = ["bundled"] } +serde = { version = "1.0.219", features = ["derive"] } +serde_json = "1.0.140" +sha2 = "0.10.8" +tempfile = "3.20.0" +thiserror = "2.0.16" +tokio = { version = "1.46.1", features = ["macros", "rt-multi-thread", "signal", "fs", "io-util"] } +tokio-stream = "0.1.17" +tokio-util = { version = "0.7.16", features = ["io"] } +tower-http = { version = "0.6.6", features = ["trace"] } +tracing = "0.1.41" +tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } +uuid = { version = "1.18.1", features = ["v4"] } + +age = "0.11.1" +secrecy = "0.10.3" +base64 = "0.22.1" +hmac = "0.12.1" diff --git a/packages/rustfs/Dockerfile b/packages/rustfs/Dockerfile new file mode 100644 index 0000000000000..4b15e005d595c --- /dev/null +++ b/packages/rustfs/Dockerfile @@ -0,0 +1,28 @@ +FROM rust:1.85-slim AS build +WORKDIR /app + +# Build deps +RUN apt-get update && apt-get install -y --no-install-recommends \ + pkg-config \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +COPY packages/rustfs/Cargo.toml packages/rustfs/Cargo.toml +COPY packages/rustfs/src packages/rustfs/src + +WORKDIR /app/packages/rustfs +RUN cargo build --release + +FROM debian:bookworm-slim +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /app +COPY --from=build /app/packages/rustfs/target/release/rustfs /usr/local/bin/rustfs + +ENV RUSTFS_PORT=8099 +ENV RUSTFS_DATA_DIR=/data +ENV RUSTFS_DB_PATH=/data/meta.db +EXPOSE 8099 + +CMD ["rustfs"] + diff --git a/packages/rustfs/README.md b/packages/rustfs/README.md new file mode 100644 index 0000000000000..ac20ad8724761 --- /dev/null +++ b/packages/rustfs/README.md @@ -0,0 +1,143 @@ +# RustFS (OpenClaw) + +`rustfs` is a small, self-hosted file archive service intended to run alongside the OpenClaw deep-memory stack. + +## Goals (MVP) + +- Ingest session files/attachments into a durable volume +- Provide file listing/search by tenant + basic metadata filters +- Support large files via streaming multipart uploads +- Optional encryption at rest using an age passphrase (`RUSTFS_MASTER_KEY`) + +## HTTP API + +All APIs are namespaced under `/v1`. + +### Upload + +`POST /v1/files` (multipart form-data) + +Fields: + +- `file` (required): file bytes +- `tenant_id` (optional): tenant hint when auth is disabled +- `session_id` (optional) +- `source` (optional) + +Response: + +```json +{ "ok": true, "file_id": "", "sha256": "", "size": 1234, "encrypted": false } +``` + +### Search/list + +`GET /v1/files?tenant_id=...&session_id=...&q=...&mime=...&limit=50` + +Response: + +```json +{ "ok": true, "items": [{ "file_id": "...", "tenant_id": "...", "filename": "...", "size": 123 }] } +``` + +### Extract jobs (worker) + +RustFS stores a per-file extraction/indexing status (`extract_status`) and optional `annotations` used by downstream semantic retrieval. +Workers are expected to **merge** annotations (preserve `openclaw_ingest` hints) and may write a structured `schema_version=1` envelope for extraction + deep-memory traceability. + +#### Claim (recommended; atomic leasing) + +`POST /v1/files/claim_extract` + +Body: + +```json +{ "tenant_id": "default", "limit": 25, "lease_ms": 300000 } +``` + +Behavior: + +- Atomically selects files that are pending (`extract_status` missing or `pending`) or stuck in `processing` past the lease window +- Marks them as `processing` (updates `extract_updated_at_ms`, increments `extract_attempt`) +- Returns the claimed file metadata (including `annotations` fields) + +This avoids duplicate processing when multiple workers are running. + +#### Pending list (legacy; non-atomic) + +`GET /v1/files/pending_extract?tenant_id=...&limit=25&lease_ms=300000` + +Note: this endpoint only lists candidates; it does not claim them atomically. Prefer `claim_extract`. + +#### Update annotations + +`POST /v1/files/:file_id/annotations` + +Body: + +```json +{ "annotations": { "any": "json" }, "source": "rustfs-worker" } +``` + +#### Update extract status + +`POST /v1/files/:file_id/extract_status` + +Body: + +```json +{ "status": "indexed", "error": "" } +``` + +### Metadata + +`GET /v1/files/:file_id/meta` + +### Tombstone (revoke) + +`POST /v1/files/:file_id/tombstone` + +Body: + +```json +{ "reason": "user_revoked" } +``` + +### Download + +`GET /v1/files/:file_id` + +### List tombstoned files (worker) + +`GET /v1/files/tombstoned?tenant_id=...&since_ms=0&limit=100` + +### Public share link (signed, short-lived) + +`POST /v1/files/:file_id/link` + +Body: + +```json +{ "ttl_seconds": 300 } +``` + +Response includes: + +- `path`: relative public download path +- `url`: absolute URL when `RUSTFS_PUBLIC_BASE_URL` is set + +Public download: + +`GET /v1/public/download?token=...` + +## Configuration + +- `RUSTFS_PORT` (default `8099`) +- `RUSTFS_DATA_DIR` (default `/data`) +- `RUSTFS_DB_PATH` (default `/data/meta.db`) +- `RUSTFS_REQUIRE_API_KEY` (default `true`) +- `RUSTFS_API_KEYS_JSON`: JSON array of `{ "key": "...", "tenant_id": "...", "role": "..." }` +- `RUSTFS_MASTER_KEY` (optional): enables encryption at rest for newly ingested files +- `RUSTFS_SIGNING_KEY` (required for share links): HMAC signing secret +- `RUSTFS_PUBLIC_BASE_URL` (optional): used to return absolute share URLs +- `RUSTFS_AUDIT_LOG_PATH` (optional): JSONL audit log path diff --git a/packages/rustfs/src/main.rs b/packages/rustfs/src/main.rs new file mode 100644 index 0000000000000..634233f163b50 --- /dev/null +++ b/packages/rustfs/src/main.rs @@ -0,0 +1,1758 @@ +use std::{ + collections::HashMap, + io::Read, + iter, + path::{Path, PathBuf}, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use age::secrecy::SecretString; +use axum::{ + body::Body, + extract::{Multipart, Query, State}, + http::{header, HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + routing::{get, post}, + Json, Router, +}; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use base64::Engine as _; +use bytes::Bytes; +use hmac::{Hmac, Mac}; +use rusqlite::{params, Connection}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use thiserror::Error; +use tokio::{fs, net::TcpListener}; +use tokio_stream::wrappers::ReceiverStream; +use tokio_util::io::ReaderStream; +use tower_http::trace::TraceLayer; +use tracing::{info, warn}; +use tokio::io::AsyncWriteExt; + +#[derive(Clone)] +struct AppState { + data_dir: PathBuf, + db_path: PathBuf, + require_api_key: bool, + api_keys: Arc>, + master_key: Option, + signing_key: Option>, + public_base_url: Option, + audit_log_path: Option, +} + +#[derive(Clone, Debug, Deserialize)] +struct ApiKey { + #[allow(dead_code)] + key: String, + tenant_id: String, + #[allow(dead_code)] + role: Option, +} + +#[derive(Clone, Debug)] +struct AuthContext { + tenant_id: String, + role: String, + key_id: String, +} + +#[derive(Debug, Serialize)] +struct ErrorBody { + error: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + message: Option, +} + +#[derive(Debug, Deserialize)] +struct SearchQuery { + tenant_id: Option, + session_id: Option, + q: Option, + mime: Option, + extract_status: Option, + limit: Option, +} + +#[derive(Debug, Serialize)] +struct FileMeta { + file_id: String, + tenant_id: String, + session_id: Option, + filename: String, + mime: Option, + size: i64, + sha256: String, + created_at_ms: i64, + source: Option, + encrypted: bool, + extract_status: Option, + extract_updated_at_ms: Option, + extract_attempt: Option, + extract_error: Option, + annotations: Option, +} + +#[derive(Debug, Serialize)] +struct SearchResponse { + ok: bool, + items: Vec, +} + +#[derive(Debug, Serialize)] +struct IngestResponse { + ok: bool, + file_id: String, + sha256: String, + size: i64, + encrypted: bool, +} + +#[derive(Debug, Deserialize)] +struct LinkRequest { + ttl_seconds: Option, +} + +#[derive(Debug, Serialize)] +struct LinkResponse { + ok: bool, + token: String, + path: String, + url: Option, + expires_at_ms: i64, +} + +#[derive(Debug, Deserialize)] +struct TombstoneRequest { + reason: Option, +} + +#[derive(Debug, Serialize)] +struct TombstoneResponse { + ok: bool, + file_id: String, + tombstoned: bool, +} + +#[derive(Error, Debug)] +enum AppError { + #[error("unauthorized")] + Unauthorized, + #[error("forbidden")] + Forbidden, + #[error("invalid_request: {0}")] + InvalidRequest(String), + #[error("not_found")] + NotFound, + #[error("io: {0}")] + Io(#[from] std::io::Error), + #[error("db: {0}")] + Db(String), + #[error("crypto: {0}")] + Crypto(String), +} + +impl IntoResponse for AppError { + fn into_response(self) -> Response { + let (status, body) = match &self { + AppError::Unauthorized => ( + StatusCode::UNAUTHORIZED, + ErrorBody { + error: "unauthorized", + message: None, + }, + ), + AppError::Forbidden => ( + StatusCode::FORBIDDEN, + ErrorBody { + error: "forbidden", + message: None, + }, + ), + AppError::InvalidRequest(msg) => ( + StatusCode::BAD_REQUEST, + ErrorBody { + error: "invalid_request", + message: Some(msg.clone()), + }, + ), + AppError::NotFound => ( + StatusCode::NOT_FOUND, + ErrorBody { + error: "not_found", + message: None, + }, + ), + _ => ( + StatusCode::INTERNAL_SERVER_ERROR, + ErrorBody { + error: "internal_error", + message: Some(self.to_string()), + }, + ), + }; + (status, Json(body)).into_response() + } +} + +fn now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or(Duration::from_secs(0)) + .as_millis() as i64 +} + +fn normalize_role(role: Option<&str>) -> String { + let r = role.unwrap_or("admin").trim().to_lowercase(); + match r.as_str() { + "reader" | "writer" | "admin" => r, + _ => "admin".to_string(), + } +} + +fn key_id_from_raw(raw_key: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(raw_key.as_bytes()); + let digest = hasher.finalize(); + hex::encode(&digest[..8]) +} + +fn auth_from_headers( + state: &AppState, + headers: &HeaderMap, + tenant_hint: Option<&str>, +) -> Result { + if !state.require_api_key { + if let Some(t) = tenant_hint { + let trimmed = t.trim(); + if !trimmed.is_empty() { + return Ok(AuthContext { + tenant_id: trimmed.to_string(), + role: "admin".to_string(), + key_id: "dev".to_string(), + }); + } + } + return Ok(AuthContext { + tenant_id: "default".to_string(), + role: "admin".to_string(), + key_id: "dev".to_string(), + }); + } + let key = headers + .get("x-api-key") + .and_then(|v| v.to_str().ok()) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .ok_or(AppError::Unauthorized)?; + let entry = state.api_keys.get(&key).ok_or(AppError::Unauthorized)?; + Ok(AuthContext { + tenant_id: entry.tenant_id.clone(), + role: normalize_role(entry.role.as_deref()), + key_id: key_id_from_raw(&key), + }) +} + +async fn init_db(db_path: &Path) -> Result<(), AppError> { + let db_path = db_path.to_path_buf(); + tokio::task::spawn_blocking(move || -> Result<(), AppError> { + let conn = Connection::open(db_path).map_err(|e| AppError::Db(e.to_string()))?; + conn.execute_batch( + r#" +CREATE TABLE IF NOT EXISTS files ( + file_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + session_id TEXT, + filename TEXT NOT NULL, + mime TEXT, + size INTEGER NOT NULL, + sha256 TEXT NOT NULL, + created_at_ms INTEGER NOT NULL, + source TEXT, + encrypted INTEGER NOT NULL, + storage_path TEXT NOT NULL, + deleted_at_ms INTEGER, + extract_status TEXT, + extract_updated_at_ms INTEGER, + extract_attempt INTEGER, + extract_error TEXT, + annotations_json TEXT +); +CREATE INDEX IF NOT EXISTS idx_files_tenant_created ON files(tenant_id, created_at_ms DESC); +CREATE INDEX IF NOT EXISTS idx_files_tenant_session ON files(tenant_id, session_id); +CREATE INDEX IF NOT EXISTS idx_files_tenant_filename ON files(tenant_id, filename); +"#, + ) + .map_err(|e| AppError::Db(e.to_string()))?; + // Back-compat: older DBs might not have deleted_at_ms. + let _ = conn.execute("ALTER TABLE files ADD COLUMN deleted_at_ms INTEGER", []); + let _ = conn.execute("ALTER TABLE files ADD COLUMN extract_status TEXT", []); + let _ = conn.execute("ALTER TABLE files ADD COLUMN extract_updated_at_ms INTEGER", []); + let _ = conn.execute("ALTER TABLE files ADD COLUMN extract_attempt INTEGER", []); + let _ = conn.execute("ALTER TABLE files ADD COLUMN extract_error TEXT", []); + let _ = conn.execute("ALTER TABLE files ADD COLUMN annotations_json TEXT", []); + Ok(()) + }) + .await + .map_err(|e| AppError::Db(e.to_string()))??; + Ok(()) +} + +#[derive(Debug, Serialize)] +struct AuditEntry<'a> { + id: String, + ts_ms: i64, + action: &'a str, + tenant_id: &'a str, + key_id: Option<&'a str>, + request_id: Option<&'a str>, + file_id: Option<&'a str>, + extra: serde_json::Value, +} + +async fn append_audit( + state: &AppState, + entry: AuditEntry<'_>, +) { + let path = match state.audit_log_path.as_ref() { + Some(p) => p, + None => return, + }; + let parent = match path.parent() { + Some(p) => p, + None => return, + }; + if fs::create_dir_all(parent).await.is_err() { + return; + } + let line = match serde_json::to_string(&entry) { + Ok(s) => s, + Err(_) => return, + }; + let mut file = match fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + .await + { + Ok(f) => f, + Err(_) => return, + }; + let _ = file.write_all(format!("{line}\n").as_bytes()).await; +} + +async fn with_conn(state: &AppState, f: impl FnOnce(&Connection) -> Result + Send + 'static) -> Result +where + T: Send + 'static, +{ + let path = state.db_path.clone(); + tokio::task::spawn_blocking(move || { + let conn = Connection::open(path).map_err(|e| AppError::Db(e.to_string()))?; + f(&conn) + }) + .await + .map_err(|e| AppError::Db(e.to_string()))? +} + +async fn health() -> impl IntoResponse { + (StatusCode::OK, Json(serde_json::json!({ "ok": true }))) +} + +async fn readyz(State(state): State) -> Result { + init_db(&state.db_path).await?; + Ok((StatusCode::OK, Json(serde_json::json!({ "ok": true })))) +} + +fn assert_can_read(auth: &AuthContext) -> Result<(), AppError> { + match auth.role.as_str() { + "reader" | "writer" | "admin" => Ok(()), + _ => Err(AppError::Forbidden), + } +} + +fn assert_can_write(auth: &AuthContext) -> Result<(), AppError> { + match auth.role.as_str() { + "writer" | "admin" => Ok(()), + _ => Err(AppError::Forbidden), + } +} + +async fn ingest( + State(state): State, + headers: HeaderMap, + mut multipart: Multipart, +) -> Result { + let mut tenant_hint: Option = None; + let mut session_id: Option = None; + let mut source: Option = None; + let mut filename: Option = None; + let mut mime: Option = None; + let mut tmp_path: Option = None; + let mut sha = Sha256::new(); + let mut size: i64 = 0; + + fs::create_dir_all(state.data_dir.join("tmp")).await?; + + while let Some(field) = multipart + .next_field() + .await + .map_err(|e| AppError::InvalidRequest(e.to_string()))? + { + let name = field.name().unwrap_or("").to_string(); + match name.as_str() { + "tenant_id" => { + let v = field + .text() + .await + .map_err(|e| AppError::InvalidRequest(e.to_string()))?; + tenant_hint = Some(v); + } + "session_id" => { + let v = field + .text() + .await + .map_err(|e| AppError::InvalidRequest(e.to_string()))?; + let trimmed = v.trim(); + session_id = if trimmed.is_empty() { + None + } else { + Some(trimmed.to_string()) + }; + } + "source" => { + let v = field + .text() + .await + .map_err(|e| AppError::InvalidRequest(e.to_string()))?; + let trimmed = v.trim(); + source = if trimmed.is_empty() { + None + } else { + Some(trimmed.to_string()) + }; + } + "file" => { + filename = field.file_name().map(|s| s.to_string()); + mime = field.content_type().map(|m| m.to_string()); + + let tmp = state + .data_dir + .join("tmp") + .join(format!("upload-{}.bin", uuid::Uuid::new_v4())); + let mut out = fs::File::create(&tmp).await?; + let mut stream = field; + while let Some(chunk) = stream + .chunk() + .await + .map_err(|e| AppError::InvalidRequest(e.to_string()))? + { + sha.update(&chunk); + size += chunk.len() as i64; + out.write_all(&chunk).await?; + } + out.flush().await?; + tmp_path = Some(tmp); + } + _ => { + // ignore unknown fields + } + } + } + + let request_id = headers.get("x-request-id").and_then(|v| v.to_str().ok()); + let auth = auth_from_headers(&state, &headers, tenant_hint.as_deref())?; + assert_can_write(&auth)?; + let tenant_id = auth.tenant_id.clone(); + let tmp = tmp_path.ok_or_else(|| AppError::InvalidRequest("missing multipart field: file".to_string()))?; + let filename = filename.unwrap_or_else(|| "file".to_string()); + let sha256 = hex::encode(sha.finalize()); + let file_id = sha256.clone(); + + let tenant_dir = state.data_dir.join("objects").join(&tenant_id); + fs::create_dir_all(&tenant_dir).await?; + + let created_at_ms = now_ms(); + let encrypted = state.master_key.is_some(); + let final_path_plain = tenant_dir.join(&file_id); + let final_path = if encrypted { + tenant_dir.join(format!("{file_id}.age")) + } else { + final_path_plain.clone() + }; + + // Insert-or-return existing by (tenant_id, file_id) + let tenant_id_for_existing = tenant_id.clone(); + let file_id_for_existing = file_id.clone(); + let existing = with_conn(&state, move |conn| { + let mut stmt = conn + .prepare( + "SELECT file_id, sha256, size, encrypted, storage_path FROM files WHERE tenant_id=?1 AND file_id=?2 AND deleted_at_ms IS NULL", + ) + .map_err(|e| AppError::Db(e.to_string()))?; + let mut rows = stmt + .query(params![tenant_id_for_existing, file_id_for_existing]) + .map_err(|e| AppError::Db(e.to_string()))?; + if let Some(row) = rows.next().map_err(|e| AppError::Db(e.to_string()))? { + let file_id: String = row.get(0).map_err(|e| AppError::Db(e.to_string()))?; + let sha256: String = row.get(1).map_err(|e| AppError::Db(e.to_string()))?; + let size: i64 = row.get(2).map_err(|e| AppError::Db(e.to_string()))?; + let encrypted: i64 = row.get(3).map_err(|e| AppError::Db(e.to_string()))?; + let _storage_path: String = row.get(4).map_err(|e| AppError::Db(e.to_string()))?; + return Ok(Some((file_id, sha256, size, encrypted != 0))); + } + Ok(None) + }) + .await?; + + if let Some((file_id, sha256, size, encrypted)) = existing { + // Best-effort cleanup tmp + let _ = fs::remove_file(&tmp).await; + append_audit( + &state, + AuditEntry { + id: uuid::Uuid::new_v4().to_string(), + ts_ms: now_ms(), + action: "ingest", + tenant_id: &tenant_id, + key_id: Some(&auth.key_id), + request_id, + file_id: Some(&file_id), + extra: serde_json::json!({ "dedup": true, "size": size, "encrypted": encrypted }), + }, + ) + .await; + return Ok(( + StatusCode::OK, + Json(IngestResponse { + ok: true, + file_id, + sha256, + size, + encrypted, + }), + )); + } + + // Move to final location, encrypt if configured. + if encrypted { + // Write plaintext to deterministic path first (temp name), then encrypt to .age and delete plaintext. + fs::rename(&tmp, &final_path_plain).await?; + + let in_path = final_path_plain.clone(); + let out_path = final_path.clone(); + let key = state.master_key.clone().ok_or_else(|| AppError::Crypto("missing master key".to_string()))?; + tokio::task::spawn_blocking(move || -> Result<(), AppError> { + let input = std::fs::File::open(&in_path)?; + let output = std::fs::File::create(&out_path)?; + let encryptor = age::Encryptor::with_user_passphrase(key); + let mut writer = encryptor + .wrap_output(output) + .map_err(|e| AppError::Crypto(e.to_string()))?; + let mut reader = std::io::BufReader::new(input); + std::io::copy(&mut reader, &mut writer)?; + writer.finish().map_err(|e| AppError::Crypto(e.to_string()))?; + Ok(()) + }) + .await + .map_err(|e| AppError::Crypto(e.to_string()))??; + + // Remove plaintext + fs::remove_file(&final_path_plain).await?; + } else { + fs::rename(&tmp, &final_path).await?; + } + + let storage_path = final_path + .strip_prefix(&state.data_dir) + .unwrap_or(&final_path) + .to_string_lossy() + .to_string(); + + let tenant_id_for_db = tenant_id.clone(); + let session_id_for_db = session_id.clone(); + let filename_for_db = filename.clone(); + let mime_for_db = mime.clone(); + let source_for_db = source.clone(); + let sha256_for_db = sha256.clone(); + let file_id_for_db = file_id.clone(); + let encrypted_i = if encrypted { 1 } else { 0 }; + let extract_status_for_db = "pending".to_string(); + let extract_updated_at_for_db = now_ms(); + let extract_attempt_for_db = 0i64; + with_conn(&state, move |conn| { + conn.execute( + "INSERT INTO files(file_id, tenant_id, session_id, filename, mime, size, sha256, created_at_ms, source, encrypted, storage_path, extract_status, extract_updated_at_ms, extract_attempt) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)", + params![ + file_id_for_db, + tenant_id_for_db, + session_id_for_db, + filename_for_db, + mime_for_db, + size, + sha256_for_db, + created_at_ms, + source_for_db, + encrypted_i, + storage_path, + extract_status_for_db, + extract_updated_at_for_db, + extract_attempt_for_db, + ], + ) + .map_err(|e| AppError::Db(e.to_string()))?; + Ok(()) + }) + .await?; + + append_audit( + &state, + AuditEntry { + id: uuid::Uuid::new_v4().to_string(), + ts_ms: now_ms(), + action: "ingest", + tenant_id: &tenant_id, + key_id: Some(&auth.key_id), + request_id, + file_id: Some(&file_id), + extra: serde_json::json!({ "dedup": false, "size": size, "encrypted": encrypted }), + }, + ) + .await; + + Ok(( + StatusCode::OK, + Json(IngestResponse { + ok: true, + file_id, + sha256, + size, + encrypted, + }), + )) +} + +async fn search( + State(state): State, + headers: HeaderMap, + Query(q): Query, +) -> Result { + let auth = auth_from_headers(&state, &headers, q.tenant_id.as_deref())?; + assert_can_read(&auth)?; + let tenant_id = auth.tenant_id; + let limit = q.limit.unwrap_or(50).clamp(1, 200) as i64; + let session_id = q.session_id.clone().filter(|s| !s.trim().is_empty()); + let query_text = q.q.clone().filter(|s| !s.trim().is_empty()); + let mime = q.mime.clone().filter(|s| !s.trim().is_empty()); + let extract_status = q.extract_status.clone().filter(|s| !s.trim().is_empty()); + + let tenant_id_db = tenant_id.clone(); + let items = with_conn(&state, move |conn| -> Result, AppError> { + let mut sql = String::from( + "SELECT file_id, tenant_id, session_id, filename, mime, size, sha256, created_at_ms, source, encrypted, extract_status, extract_updated_at_ms, extract_attempt, extract_error, annotations_json + FROM files WHERE tenant_id=?1 AND deleted_at_ms IS NULL", + ); + let mut args: Vec = vec![tenant_id_db.clone().into()]; + + if let Some(sid) = &session_id { + sql.push_str(" AND session_id=?"); + args.push(sid.clone().into()); + } + if let Some(m) = &mime { + sql.push_str(" AND mime=?"); + args.push(m.clone().into()); + } + if let Some(text) = &query_text { + sql.push_str(" AND (filename LIKE ? OR file_id LIKE ? OR sha256 LIKE ?)"); + let like = format!("%{}%", text); + args.push(like.clone().into()); + args.push(like.clone().into()); + args.push(like.into()); + } + if let Some(status) = &extract_status { + sql.push_str(" AND extract_status=?"); + args.push(status.clone().into()); + } + sql.push_str(" ORDER BY created_at_ms DESC LIMIT ?"); + args.push(limit.into()); + + let mut stmt = conn.prepare(&sql).map_err(|e| AppError::Db(e.to_string()))?; + let mut rows = stmt + .query(rusqlite::params_from_iter(args)) + .map_err(|e| AppError::Db(e.to_string()))?; + let mut out = Vec::new(); + while let Some(row) = rows.next().map_err(|e| AppError::Db(e.to_string()))? { + let annotations_raw: Option = row.get(14).ok(); + let annotations = annotations_raw + .as_deref() + .and_then(|s| serde_json::from_str::(s).ok()); + out.push(FileMeta { + file_id: row.get(0).map_err(|e| AppError::Db(e.to_string()))?, + tenant_id: row.get(1).map_err(|e| AppError::Db(e.to_string()))?, + session_id: row.get(2).map_err(|e| AppError::Db(e.to_string()))?, + filename: row.get(3).map_err(|e| AppError::Db(e.to_string()))?, + mime: row.get(4).map_err(|e| AppError::Db(e.to_string()))?, + size: row.get(5).map_err(|e| AppError::Db(e.to_string()))?, + sha256: row.get(6).map_err(|e| AppError::Db(e.to_string()))?, + created_at_ms: row.get(7).map_err(|e| AppError::Db(e.to_string()))?, + source: row.get(8).map_err(|e| AppError::Db(e.to_string()))?, + encrypted: { + let v: i64 = row.get(9).map_err(|e| AppError::Db(e.to_string()))?; + v != 0 + }, + extract_status: row.get(10).ok(), + extract_updated_at_ms: row.get(11).ok(), + extract_attempt: row.get(12).ok(), + extract_error: row.get(13).ok(), + annotations, + }); + } + Ok(out) + }) + .await?; + + Ok((StatusCode::OK, Json(SearchResponse { ok: true, items }))) +} + +#[derive(Debug, Deserialize)] +struct PathParams { + file_id: String, +} + +#[derive(Debug, Deserialize)] +struct PendingQuery { + tenant_id: Option, + limit: Option, + // Re-lease "processing" jobs if stuck longer than this. + lease_ms: Option, +} + +#[derive(Debug, Serialize)] +struct PendingResponse { + ok: bool, + items: Vec, +} + +#[derive(Debug, Deserialize)] +struct ClaimExtractRequest { + tenant_id: Option, + limit: Option, + // Re-lease "processing" jobs if stuck longer than this. + lease_ms: Option, +} + +#[derive(Debug, Serialize)] +struct ClaimExtractResponse { + ok: bool, + items: Vec, + claimed_at_ms: i64, +} + +#[derive(Debug, Deserialize)] +struct AnnotationsRequest { + // Keep this opaque and LLM-friendly: semantics are application-defined. + annotations: serde_json::Value, + source: Option, +} + +#[derive(Debug, Serialize)] +struct AnnotationsResponse { + ok: bool, + file_id: String, + updated_at_ms: i64, +} + +#[derive(Debug, Deserialize)] +struct ExtractStatusRequest { + status: String, + error: Option, +} + +#[derive(Debug, Serialize)] +struct ExtractStatusResponse { + ok: bool, + file_id: String, + status: String, +} + +#[derive(Debug, Deserialize)] +struct TombstonedQuery { + tenant_id: Option, + since_ms: Option, + limit: Option, +} + +#[derive(Debug, Serialize)] +struct TombstonedItem { + file_id: String, + deleted_at_ms: i64, +} + +#[derive(Debug, Serialize)] +struct TombstonedResponse { + ok: bool, + items: Vec, +} + +async fn meta( + State(state): State, + headers: HeaderMap, + axum::extract::Path(path): axum::extract::Path, +) -> Result { + let auth = auth_from_headers(&state, &headers, None)?; + assert_can_read(&auth)?; + let tenant_id = auth.tenant_id; + let file_id = path.file_id.trim().to_string(); + if file_id.is_empty() { + return Err(AppError::InvalidRequest("file_id required".to_string())); + } + let tenant_id_db = tenant_id.clone(); + let out = with_conn(&state, move |conn| -> Result, AppError> { + let mut stmt = conn + .prepare( + "SELECT file_id, tenant_id, session_id, filename, mime, size, sha256, created_at_ms, source, encrypted, extract_status, extract_updated_at_ms, extract_attempt, extract_error, annotations_json + FROM files WHERE tenant_id=?1 AND file_id=?2 AND deleted_at_ms IS NULL", + ) + .map_err(|e| AppError::Db(e.to_string()))?; + let mut rows = stmt + .query(params![tenant_id_db, file_id]) + .map_err(|e| AppError::Db(e.to_string()))?; + if let Some(row) = rows.next().map_err(|e| AppError::Db(e.to_string()))? { + let annotations_raw: Option = row.get(14).ok(); + let annotations = annotations_raw + .as_deref() + .and_then(|s| serde_json::from_str::(s).ok()); + return Ok(Some(FileMeta { + file_id: row.get(0).map_err(|e| AppError::Db(e.to_string()))?, + tenant_id: row.get(1).map_err(|e| AppError::Db(e.to_string()))?, + session_id: row.get(2).map_err(|e| AppError::Db(e.to_string()))?, + filename: row.get(3).map_err(|e| AppError::Db(e.to_string()))?, + mime: row.get(4).map_err(|e| AppError::Db(e.to_string()))?, + size: row.get(5).map_err(|e| AppError::Db(e.to_string()))?, + sha256: row.get(6).map_err(|e| AppError::Db(e.to_string()))?, + created_at_ms: row.get(7).map_err(|e| AppError::Db(e.to_string()))?, + source: row.get(8).map_err(|e| AppError::Db(e.to_string()))?, + encrypted: { + let v: i64 = row.get(9).map_err(|e| AppError::Db(e.to_string()))?; + v != 0 + }, + extract_status: row.get(10).ok(), + extract_updated_at_ms: row.get(11).ok(), + extract_attempt: row.get(12).ok(), + extract_error: row.get(13).ok(), + annotations, + })); + } + Ok(None) + }) + .await?; + match out { + Some(v) => Ok((StatusCode::OK, Json(v))), + None => Err(AppError::NotFound), + } +} + +async fn pending_extract( + State(state): State, + headers: HeaderMap, + Query(q): Query, +) -> Result { + let auth = auth_from_headers(&state, &headers, q.tenant_id.as_deref())?; + assert_can_read(&auth)?; + let tenant_id = auth.tenant_id; + let limit = q.limit.unwrap_or(25).clamp(1, 200) as i64; + let lease_ms = q.lease_ms.unwrap_or(300_000).clamp(5_000, 86_400_000) as i64; + let now = now_ms(); + let lease_before_ms = now - lease_ms; + + let tenant_id_db = tenant_id.clone(); + let items = with_conn(&state, move |conn| -> Result, AppError> { + let mut stmt = conn + .prepare( + "SELECT file_id, tenant_id, session_id, filename, mime, size, sha256, created_at_ms, source, encrypted, extract_status, extract_updated_at_ms, extract_attempt, extract_error, annotations_json + FROM files + WHERE tenant_id=?1 AND deleted_at_ms IS NULL AND ( + extract_status IS NULL + OR extract_status='pending' + OR (extract_status='processing' AND COALESCE(extract_updated_at_ms, 0) <= ?3) + ) + ORDER BY created_at_ms ASC + LIMIT ?2", + ) + .map_err(|e| AppError::Db(e.to_string()))?; + let mut rows = stmt + .query(params![tenant_id_db, limit, lease_before_ms]) + .map_err(|e| AppError::Db(e.to_string()))?; + let mut out = Vec::new(); + while let Some(row) = rows.next().map_err(|e| AppError::Db(e.to_string()))? { + let annotations_raw: Option = row.get(14).ok(); + let annotations = annotations_raw + .as_deref() + .and_then(|s| serde_json::from_str::(s).ok()); + out.push(FileMeta { + file_id: row.get(0).map_err(|e| AppError::Db(e.to_string()))?, + tenant_id: row.get(1).map_err(|e| AppError::Db(e.to_string()))?, + session_id: row.get(2).map_err(|e| AppError::Db(e.to_string()))?, + filename: row.get(3).map_err(|e| AppError::Db(e.to_string()))?, + mime: row.get(4).map_err(|e| AppError::Db(e.to_string()))?, + size: row.get(5).map_err(|e| AppError::Db(e.to_string()))?, + sha256: row.get(6).map_err(|e| AppError::Db(e.to_string()))?, + created_at_ms: row.get(7).map_err(|e| AppError::Db(e.to_string()))?, + source: row.get(8).map_err(|e| AppError::Db(e.to_string()))?, + encrypted: { + let v: i64 = row.get(9).map_err(|e| AppError::Db(e.to_string()))?; + v != 0 + }, + extract_status: row.get(10).ok(), + extract_updated_at_ms: row.get(11).ok(), + extract_attempt: row.get(12).ok(), + extract_error: row.get(13).ok(), + annotations, + }); + } + Ok(out) + }) + .await?; + + Ok((StatusCode::OK, Json(PendingResponse { ok: true, items }))) +} + +async fn claim_extract( + State(state): State, + headers: HeaderMap, + Json(req): Json, +) -> Result { + let auth = auth_from_headers(&state, &headers, req.tenant_id.as_deref())?; + // Claiming is a mutating operation (leases work); require write. + assert_can_write(&auth)?; + let tenant_id = auth.tenant_id; + let limit = req.limit.unwrap_or(25).clamp(1, 200) as i64; + let lease_ms = req.lease_ms.unwrap_or(300_000).clamp(5_000, 86_400_000) as i64; + let now = now_ms(); + let lease_before_ms = now - lease_ms; + + let tenant_id_db = tenant_id.clone(); + let items = with_conn(&state, move |conn| -> Result, AppError> { + // We want an IMMEDIATE transaction to atomically claim work across + // multiple workers. Use BEGIN IMMEDIATE to avoid requiring &mut Connection. + conn.execute_batch("BEGIN IMMEDIATE") + .map_err(|e| AppError::Db(e.to_string()))?; + + let res: Result, AppError> = (|| { + // 1) Select candidate ids inside the transaction. + let mut stmt = conn + .prepare( + "SELECT file_id + FROM files + WHERE tenant_id=?1 AND deleted_at_ms IS NULL AND ( + extract_status IS NULL + OR extract_status='pending' + OR (extract_status='processing' AND COALESCE(extract_updated_at_ms, 0) <= ?3) + ) + ORDER BY created_at_ms ASC + LIMIT ?2", + ) + .map_err(|e| AppError::Db(e.to_string()))?; + let mut rows = stmt + .query(params![tenant_id_db.clone(), limit, lease_before_ms]) + .map_err(|e| AppError::Db(e.to_string()))?; + let mut ids: Vec = Vec::new(); + while let Some(row) = rows.next().map_err(|e| AppError::Db(e.to_string()))? { + let id: String = row.get(0).map_err(|e| AppError::Db(e.to_string()))?; + if !id.trim().is_empty() { + ids.push(id); + } + } + drop(rows); + drop(stmt); + + if ids.is_empty() { + return Ok(Vec::new()); + } + + // 2) Mark them as processing (lease). + for file_id in &ids { + conn.execute( + "UPDATE files + SET extract_status='processing', + extract_updated_at_ms=?1, + extract_attempt=COALESCE(extract_attempt, 0) + 1, + extract_error='' + WHERE tenant_id=?2 AND file_id=?3 AND deleted_at_ms IS NULL", + params![now, tenant_id_db.clone(), file_id], + ) + .map_err(|e| AppError::Db(e.to_string()))?; + } + + // 3) Load and return the claimed rows. + let placeholders = ids.iter().map(|_| "?").collect::>().join(", "); + let sql = format!( + "SELECT file_id, tenant_id, session_id, filename, mime, size, sha256, created_at_ms, source, encrypted, extract_status, extract_updated_at_ms, extract_attempt, extract_error, annotations_json + FROM files + WHERE tenant_id=?1 AND deleted_at_ms IS NULL AND file_id IN ({placeholders}) + ORDER BY created_at_ms ASC" + ); + let mut args: Vec = vec![tenant_id_db.clone().into()]; + for id in &ids { + args.push(id.clone().into()); + } + let mut stmt2 = conn.prepare(&sql).map_err(|e| AppError::Db(e.to_string()))?; + let mut rows2 = stmt2 + .query(rusqlite::params_from_iter(args)) + .map_err(|e| AppError::Db(e.to_string()))?; + let mut out = Vec::new(); + while let Some(row) = rows2.next().map_err(|e| AppError::Db(e.to_string()))? { + let annotations_raw: Option = row.get(14).ok(); + let annotations = annotations_raw + .as_deref() + .and_then(|s| serde_json::from_str::(s).ok()); + out.push(FileMeta { + file_id: row.get(0).map_err(|e| AppError::Db(e.to_string()))?, + tenant_id: row.get(1).map_err(|e| AppError::Db(e.to_string()))?, + session_id: row.get(2).map_err(|e| AppError::Db(e.to_string()))?, + filename: row.get(3).map_err(|e| AppError::Db(e.to_string()))?, + mime: row.get(4).map_err(|e| AppError::Db(e.to_string()))?, + size: row.get(5).map_err(|e| AppError::Db(e.to_string()))?, + sha256: row.get(6).map_err(|e| AppError::Db(e.to_string()))?, + created_at_ms: row.get(7).map_err(|e| AppError::Db(e.to_string()))?, + source: row.get(8).map_err(|e| AppError::Db(e.to_string()))?, + encrypted: { + let v: i64 = row.get(9).map_err(|e| AppError::Db(e.to_string()))?; + v != 0 + }, + extract_status: row.get(10).ok(), + extract_updated_at_ms: row.get(11).ok(), + extract_attempt: row.get(12).ok(), + extract_error: row.get(13).ok(), + annotations, + }); + } + drop(rows2); + drop(stmt2); + + Ok(out) + })(); + + match &res { + Ok(_) => { + conn.execute_batch("COMMIT") + .map_err(|e| AppError::Db(e.to_string()))?; + } + Err(_) => { + let _ = conn.execute_batch("ROLLBACK"); + } + } + + res + }) + .await?; + + Ok(( + StatusCode::OK, + Json(ClaimExtractResponse { + ok: true, + items, + claimed_at_ms: now, + }), + )) +} + +async fn upsert_annotations( + State(state): State, + headers: HeaderMap, + axum::extract::Path(path): axum::extract::Path, + Json(req): Json, +) -> Result { + let request_id = headers.get("x-request-id").and_then(|v| v.to_str().ok()); + let auth = auth_from_headers(&state, &headers, None)?; + assert_can_write(&auth)?; + + let file_id = path.file_id.trim().to_string(); + if file_id.is_empty() { + return Err(AppError::InvalidRequest("file_id required".to_string())); + } + let now = now_ms(); + let tenant_id = auth.tenant_id.clone(); + let file_id_db = file_id.clone(); + let annotations_json = + serde_json::to_string(&req.annotations).map_err(|e| AppError::InvalidRequest(e.to_string()))?; + + let updated = with_conn(&state, move |conn| -> Result { + let n = conn + .execute( + "UPDATE files SET annotations_json=?1, extract_updated_at_ms=?2 WHERE tenant_id=?3 AND file_id=?4 AND deleted_at_ms IS NULL", + params![annotations_json, now, tenant_id, file_id_db], + ) + .map_err(|e| AppError::Db(e.to_string()))?; + Ok(n) + }) + .await?; + if updated == 0 { + return Err(AppError::NotFound); + } + + let source = req.source.unwrap_or_else(|| "unknown".to_string()); + append_audit( + &state, + AuditEntry { + id: uuid::Uuid::new_v4().to_string(), + ts_ms: now_ms(), + action: "annotations_upsert", + tenant_id: &auth.tenant_id, + key_id: Some(&auth.key_id), + request_id, + file_id: Some(&file_id), + extra: serde_json::json!({ "source": source }), + }, + ) + .await; + + Ok(( + StatusCode::OK, + Json(AnnotationsResponse { + ok: true, + file_id, + updated_at_ms: now, + }), + )) +} + +async fn set_extract_status( + State(state): State, + headers: HeaderMap, + axum::extract::Path(path): axum::extract::Path, + Json(req): Json, +) -> Result { + let request_id = headers.get("x-request-id").and_then(|v| v.to_str().ok()); + let auth = auth_from_headers(&state, &headers, None)?; + assert_can_write(&auth)?; + + let file_id = path.file_id.trim().to_string(); + if file_id.is_empty() { + return Err(AppError::InvalidRequest("file_id required".to_string())); + } + let status = req.status.trim().to_string(); + if status.is_empty() { + return Err(AppError::InvalidRequest("status required".to_string())); + } + + let now = now_ms(); + let tenant_id = auth.tenant_id.clone(); + let file_id_db = file_id.clone(); + let error = req.error.unwrap_or_default(); + let status_db = status.clone(); + let error_db = error.clone(); + let updated = with_conn(&state, move |conn| -> Result { + let n = conn + .execute( + "UPDATE files + SET extract_status=?1, + extract_updated_at_ms=?2, + extract_attempt=COALESCE(extract_attempt, 0) + 1, + extract_error=?3 + WHERE tenant_id=?4 AND file_id=?5 AND deleted_at_ms IS NULL", + params![status_db, now, error_db, tenant_id, file_id_db], + ) + .map_err(|e| AppError::Db(e.to_string()))?; + Ok(n) + }) + .await?; + if updated == 0 { + return Err(AppError::NotFound); + } + + append_audit( + &state, + AuditEntry { + id: uuid::Uuid::new_v4().to_string(), + ts_ms: now_ms(), + action: "extract_status", + tenant_id: &auth.tenant_id, + key_id: Some(&auth.key_id), + request_id, + file_id: Some(&file_id), + extra: serde_json::json!({ "status": status, "has_error": !error.is_empty() }), + }, + ) + .await; + + Ok(( + StatusCode::OK, + Json(ExtractStatusResponse { + ok: true, + file_id, + status, + }), + )) +} + +async fn tombstoned( + State(state): State, + headers: HeaderMap, + Query(q): Query, +) -> Result { + let auth = auth_from_headers(&state, &headers, q.tenant_id.as_deref())?; + assert_can_read(&auth)?; + let tenant_id = auth.tenant_id; + let limit = q.limit.unwrap_or(100).clamp(1, 200) as i64; + let since_ms = q.since_ms.unwrap_or(0).max(0); + + let tenant_id_db = tenant_id.clone(); + let items = with_conn(&state, move |conn| -> Result, AppError> { + let mut stmt = conn + .prepare( + "SELECT file_id, deleted_at_ms + FROM files + WHERE tenant_id=?1 AND deleted_at_ms IS NOT NULL AND deleted_at_ms > ?2 + ORDER BY deleted_at_ms ASC + LIMIT ?3", + ) + .map_err(|e| AppError::Db(e.to_string()))?; + let mut rows = stmt + .query(params![tenant_id_db, since_ms, limit]) + .map_err(|e| AppError::Db(e.to_string()))?; + let mut out = Vec::new(); + while let Some(row) = rows.next().map_err(|e| AppError::Db(e.to_string()))? { + out.push(TombstonedItem { + file_id: row.get(0).map_err(|e| AppError::Db(e.to_string()))?, + deleted_at_ms: row.get(1).map_err(|e| AppError::Db(e.to_string()))?, + }); + } + Ok(out) + }) + .await?; + + Ok((StatusCode::OK, Json(TombstonedResponse { ok: true, items }))) +} + +async fn download( + State(state): State, + headers: HeaderMap, + axum::extract::Path(path): axum::extract::Path, +) -> Result { + let auth = auth_from_headers(&state, &headers, None)?; + assert_can_read(&auth)?; + let tenant_id = auth.tenant_id; + let file_id = path.file_id.trim().to_string(); + if file_id.is_empty() { + return Err(AppError::InvalidRequest("file_id required".to_string())); + } + + #[derive(Debug)] + struct Row { + filename: String, + mime: Option, + encrypted: bool, + storage_path: String, + } + + let tenant_id_db = tenant_id.clone(); + let row = with_conn(&state, move |conn| -> Result, AppError> { + let mut stmt = conn + .prepare( + "SELECT filename, mime, encrypted, storage_path FROM files WHERE tenant_id=?1 AND file_id=?2 AND deleted_at_ms IS NULL", + ) + .map_err(|e| AppError::Db(e.to_string()))?; + let mut rows = stmt + .query(params![tenant_id_db, file_id]) + .map_err(|e| AppError::Db(e.to_string()))?; + if let Some(row) = rows.next().map_err(|e| AppError::Db(e.to_string()))? { + let encrypted_i: i64 = row.get(2).map_err(|e| AppError::Db(e.to_string()))?; + return Ok(Some(Row { + filename: row.get(0).map_err(|e| AppError::Db(e.to_string()))?, + mime: row.get(1).map_err(|e| AppError::Db(e.to_string()))?, + encrypted: encrypted_i != 0, + storage_path: row.get(3).map_err(|e| AppError::Db(e.to_string()))?, + })); + } + Ok(None) + }) + .await? + .ok_or(AppError::NotFound)?; + + let abs = state.data_dir.join(row.storage_path.trim_start_matches('/')); + if !abs.exists() { + return Err(AppError::NotFound); + } + + let mut headers_out = HeaderMap::new(); + headers_out.insert( + header::CONTENT_DISPOSITION, + format!("attachment; filename=\"{}\"", row.filename.replace('"', "_")) + .parse() + .unwrap(), + ); + if let Some(ct) = row.mime.as_deref() { + if let Ok(v) = ct.parse() { + headers_out.insert(header::CONTENT_TYPE, v); + } + } + + if !row.encrypted { + let file = fs::File::open(abs).await?; + let body = Body::from_stream(ReaderStream::new(file)); + return Ok((StatusCode::OK, headers_out, body)); + } + + // Encrypted: decrypt on the fly (blocking reader -> async body stream). + let key = state + .master_key + .clone() + .ok_or_else(|| AppError::Crypto("encrypted file but no master key configured".to_string()))?; + let (tx, rx) = tokio::sync::mpsc::channel::>(8); + tokio::task::spawn_blocking(move || { + let result: Result<(), std::io::Error> = (|| { + let input = std::fs::File::open(abs)?; + let decryptor = age::Decryptor::new(std::io::BufReader::new(input)) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; + let identity = age::scrypt::Identity::new(key); + let mut reader = decryptor + .decrypt(iter::once(&identity as &dyn age::Identity)) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; + + let mut buf = vec![0u8; 64 * 1024]; + loop { + let n = reader.read(&mut buf)?; + if n == 0 { + break; + } + if tx.blocking_send(Ok(Bytes::copy_from_slice(&buf[..n]))).is_err() { + break; + } + } + Ok(()) + })(); + if let Err(e) = result { + let _ = tx.blocking_send(Err(e)); + } + }); + + let body = Body::from_stream(ReceiverStream::new(rx)); + Ok((StatusCode::OK, headers_out, body)) +} + +#[derive(Debug, Serialize, Deserialize)] +struct DownloadTokenPayload { + tenant_id: String, + file_id: String, + exp_ms: i64, +} + +type HmacSha256 = Hmac; + +fn sign_token(signing_key: &[u8], payload: &DownloadTokenPayload) -> Result { + let payload_json = + serde_json::to_vec(payload).map_err(|e| AppError::InvalidRequest(e.to_string()))?; + let payload_b64 = URL_SAFE_NO_PAD.encode(payload_json); + let mut mac = HmacSha256::new_from_slice(signing_key) + .map_err(|_| AppError::InvalidRequest("invalid signing key".to_string()))?; + mac.update(payload_b64.as_bytes()); + let sig = mac.finalize().into_bytes(); + let sig_b64 = URL_SAFE_NO_PAD.encode(sig); + Ok(format!("{payload_b64}.{sig_b64}")) +} + +fn verify_token(signing_key: &[u8], token: &str) -> Result { + let parts: Vec<&str> = token.split('.').collect(); + if parts.len() != 2 { + return Err(AppError::InvalidRequest("invalid token".to_string())); + } + let payload_b64 = parts[0]; + let sig_b64 = parts[1]; + + let sig = URL_SAFE_NO_PAD + .decode(sig_b64.as_bytes()) + .map_err(|_| AppError::InvalidRequest("invalid token".to_string()))?; + + let mut mac = HmacSha256::new_from_slice(signing_key) + .map_err(|_| AppError::InvalidRequest("invalid signing key".to_string()))?; + mac.update(payload_b64.as_bytes()); + mac.verify_slice(&sig) + .map_err(|_| AppError::Unauthorized)?; + + let payload_json = URL_SAFE_NO_PAD + .decode(payload_b64.as_bytes()) + .map_err(|_| AppError::InvalidRequest("invalid token".to_string()))?; + let payload: DownloadTokenPayload = serde_json::from_slice(&payload_json) + .map_err(|_| AppError::InvalidRequest("invalid token".to_string()))?; + if payload.exp_ms <= now_ms() { + return Err(AppError::Unauthorized); + } + Ok(payload) +} + +#[derive(Debug, Deserialize)] +struct PublicDownloadQuery { + token: String, +} + +async fn create_link( + State(state): State, + headers: HeaderMap, + axum::extract::Path(path): axum::extract::Path, + Json(req): Json, +) -> Result { + let auth = auth_from_headers(&state, &headers, None)?; + assert_can_write(&auth)?; + + let signing_key = state.signing_key.as_deref().ok_or_else(|| { + AppError::InvalidRequest("RUSTFS_SIGNING_KEY is not configured".to_string()) + })?; + + let file_id = path.file_id.trim().to_string(); + if file_id.is_empty() { + return Err(AppError::InvalidRequest("file_id required".to_string())); + } + + // Ensure file exists for this tenant (avoid token generation for missing / wrong tenant) + let tenant_id_db = auth.tenant_id.clone(); + let file_id_db = file_id.clone(); + let exists = with_conn(&state, move |conn| -> Result { + let mut stmt = conn + .prepare("SELECT COUNT(*) FROM files WHERE tenant_id=?1 AND file_id=?2 AND deleted_at_ms IS NULL") + .map_err(|e| AppError::Db(e.to_string()))?; + let count: i64 = stmt + .query_row(params![tenant_id_db, file_id_db], |row| row.get(0)) + .map_err(|e| AppError::Db(e.to_string()))?; + Ok(count > 0) + }) + .await?; + if !exists { + return Err(AppError::NotFound); + } + + let ttl = req.ttl_seconds.unwrap_or(300).clamp(30, 3600) as i64; + let expires_at_ms = now_ms() + ttl * 1000; + let payload = DownloadTokenPayload { + tenant_id: auth.tenant_id.clone(), + file_id: file_id.clone(), + exp_ms: expires_at_ms, + }; + let token = sign_token(signing_key, &payload)?; + let path = format!("/v1/public/download?token={}", token); + let url = state.public_base_url.as_deref().map(|base| { + let b = base.trim_end_matches('/'); + format!("{b}{path}") + }); + let request_id = headers.get("x-request-id").and_then(|v| v.to_str().ok()); + append_audit( + &state, + AuditEntry { + id: uuid::Uuid::new_v4().to_string(), + ts_ms: now_ms(), + action: "link_create", + tenant_id: &auth.tenant_id, + key_id: Some(&auth.key_id), + request_id, + file_id: Some(&file_id), + extra: serde_json::json!({ "ttl_seconds": ttl }), + }, + ) + .await; + Ok(( + StatusCode::OK, + Json(LinkResponse { + ok: true, + token, + path, + url, + expires_at_ms, + }), + )) +} + +async fn public_download( + State(state): State, + Query(q): Query, +) -> Result { + let signing_key = state.signing_key.as_deref().ok_or_else(|| { + AppError::InvalidRequest("RUSTFS_SIGNING_KEY is not configured".to_string()) + })?; + let payload = verify_token(signing_key, q.token.trim())?; + + // Reuse existing download logic by querying metadata and streaming file. + // This endpoint bypasses API key auth but is constrained by the signed token. + let tenant_id = payload.tenant_id; + let file_id = payload.file_id; + append_audit( + &state, + AuditEntry { + id: uuid::Uuid::new_v4().to_string(), + ts_ms: now_ms(), + action: "public_download", + tenant_id: &tenant_id, + key_id: None, + request_id: None, + file_id: Some(&file_id), + extra: serde_json::json!({}), + }, + ) + .await; + + #[derive(Debug)] + struct Row { + filename: String, + mime: Option, + encrypted: bool, + storage_path: String, + } + + let tenant_id_db = tenant_id.clone(); + let file_id_db = file_id.clone(); + let row = with_conn(&state, move |conn| -> Result, AppError> { + let mut stmt = conn + .prepare( + "SELECT filename, mime, encrypted, storage_path FROM files WHERE tenant_id=?1 AND file_id=?2 AND deleted_at_ms IS NULL", + ) + .map_err(|e| AppError::Db(e.to_string()))?; + let mut rows = stmt + .query(params![tenant_id_db, file_id_db]) + .map_err(|e| AppError::Db(e.to_string()))?; + if let Some(row) = rows.next().map_err(|e| AppError::Db(e.to_string()))? { + let encrypted_i: i64 = row.get(2).map_err(|e| AppError::Db(e.to_string()))?; + return Ok(Some(Row { + filename: row.get(0).map_err(|e| AppError::Db(e.to_string()))?, + mime: row.get(1).map_err(|e| AppError::Db(e.to_string()))?, + encrypted: encrypted_i != 0, + storage_path: row.get(3).map_err(|e| AppError::Db(e.to_string()))?, + })); + } + Ok(None) + }) + .await? + .ok_or(AppError::NotFound)?; + + let abs = state.data_dir.join(row.storage_path.trim_start_matches('/')); + if !abs.exists() { + return Err(AppError::NotFound); + } + + let mut headers_out = HeaderMap::new(); + headers_out.insert( + header::CONTENT_DISPOSITION, + format!("attachment; filename=\"{}\"", row.filename.replace('"', "_")) + .parse() + .unwrap(), + ); + if let Some(ct) = row.mime.as_deref() { + if let Ok(v) = ct.parse() { + headers_out.insert(header::CONTENT_TYPE, v); + } + } + + if !row.encrypted { + let file = fs::File::open(abs).await?; + let body = Body::from_stream(ReaderStream::new(file)); + return Ok((StatusCode::OK, headers_out, body)); + } + + let key = state + .master_key + .clone() + .ok_or_else(|| AppError::Crypto("encrypted file but no master key configured".to_string()))?; + let (tx, rx) = tokio::sync::mpsc::channel::>(8); + tokio::task::spawn_blocking(move || { + let result: Result<(), std::io::Error> = (|| { + let input = std::fs::File::open(abs)?; + let decryptor = age::Decryptor::new(std::io::BufReader::new(input)) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; + let identity = age::scrypt::Identity::new(key); + let mut reader = decryptor + .decrypt(iter::once(&identity as &dyn age::Identity)) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; + + let mut buf = vec![0u8; 64 * 1024]; + loop { + let n = reader.read(&mut buf)?; + if n == 0 { + break; + } + if tx.blocking_send(Ok(Bytes::copy_from_slice(&buf[..n]))).is_err() { + break; + } + } + Ok(()) + })(); + if let Err(e) = result { + let _ = tx.blocking_send(Err(e)); + } + }); + + let body = Body::from_stream(ReceiverStream::new(rx)); + Ok((StatusCode::OK, headers_out, body)) +} + +async fn tombstone( + State(state): State, + headers: HeaderMap, + axum::extract::Path(path): axum::extract::Path, + Json(req): Json, +) -> Result { + let request_id = headers.get("x-request-id").and_then(|v| v.to_str().ok()); + let auth = auth_from_headers(&state, &headers, None)?; + assert_can_write(&auth)?; + let file_id = path.file_id.trim().to_string(); + if file_id.is_empty() { + return Err(AppError::InvalidRequest("file_id required".to_string())); + } + let tenant_id = auth.tenant_id.clone(); + let file_id_db = file_id.clone(); + let ts = now_ms(); + let updated = with_conn(&state, move |conn| -> Result { + let n = conn + .execute( + "UPDATE files SET deleted_at_ms=?1 WHERE tenant_id=?2 AND file_id=?3 AND deleted_at_ms IS NULL", + params![ts, tenant_id, file_id_db], + ) + .map_err(|e| AppError::Db(e.to_string()))?; + Ok(n) + }) + .await?; + let tombstoned = updated > 0; + append_audit( + &state, + AuditEntry { + id: uuid::Uuid::new_v4().to_string(), + ts_ms: now_ms(), + action: "tombstone", + tenant_id: &auth.tenant_id, + key_id: Some(&auth.key_id), + request_id, + file_id: Some(&file_id), + extra: serde_json::json!({ "reason": req.reason, "tombstoned": tombstoned }), + }, + ) + .await; + Ok(( + StatusCode::OK, + Json(TombstoneResponse { + ok: true, + file_id, + tombstoned, + }), + )) +} + +fn parse_api_keys_json(raw: &str) -> HashMap { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return HashMap::new(); + } + let parsed: serde_json::Value = match serde_json::from_str(trimmed) { + Ok(v) => v, + Err(_) => return HashMap::new(), + }; + let mut map = HashMap::new(); + let arr = parsed.as_array().cloned().unwrap_or_default(); + for item in arr { + let key = item.get("key").and_then(|v| v.as_str()).unwrap_or("").trim().to_string(); + let tenant_id = item + .get("tenant_id") + .and_then(|v| v.as_str()) + .unwrap_or("") + .trim() + .to_string(); + let role = item.get("role").and_then(|v| v.as_str()).map(|s| s.to_string()); + if key.is_empty() || tenant_id.is_empty() { + continue; + } + map.insert( + key.clone(), + ApiKey { + key, + tenant_id, + role, + }, + ); + } + map +} + +#[tokio::main] +async fn main() -> Result<(), anyhow::Error> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| "rustfs=info,tower_http=warn".into()), + ) + .init(); + + let port: u16 = std::env::var("RUSTFS_PORT") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(8099); + let data_dir = std::env::var("RUSTFS_DATA_DIR").unwrap_or_else(|_| "/data".to_string()); + let db_path = std::env::var("RUSTFS_DB_PATH").unwrap_or_else(|_| "/data/meta.db".to_string()); + let require_api_key = std::env::var("RUSTFS_REQUIRE_API_KEY") + .ok() + .map(|v| v.trim().to_lowercase() == "true" || v.trim() == "1") + .unwrap_or(true); + let api_keys_json = std::env::var("RUSTFS_API_KEYS_JSON").unwrap_or_default(); + let master_key_raw = std::env::var("RUSTFS_MASTER_KEY").ok().map(|v| v.trim().to_string()); + let master_key = master_key_raw + .as_deref() + .filter(|s| !s.is_empty()) + .map(|s| SecretString::from(s.to_string())); + let signing_key = std::env::var("RUSTFS_SIGNING_KEY") + .ok() + .map(|v| v.trim().as_bytes().to_vec()) + .filter(|v| !v.is_empty()); + let public_base_url = std::env::var("RUSTFS_PUBLIC_BASE_URL") + .ok() + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()); + let audit_log_path = std::env::var("RUSTFS_AUDIT_LOG_PATH") + .ok() + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) + .map(PathBuf::from); + + let state = AppState { + data_dir: PathBuf::from(data_dir), + db_path: PathBuf::from(db_path), + require_api_key, + api_keys: Arc::new(parse_api_keys_json(&api_keys_json)), + master_key, + signing_key, + public_base_url, + audit_log_path, + }; + + fs::create_dir_all(&state.data_dir).await?; + init_db(&state.db_path).await.map_err(|e| anyhow::anyhow!(e.to_string()))?; + + if state.require_api_key && state.api_keys.is_empty() { + warn!("RUSTFS_REQUIRE_API_KEY=true but RUSTFS_API_KEYS_JSON is empty; all requests will be unauthorized"); + } + info!( + "rustfs starting: port={} data_dir={} db_path={} encryption={}", + port, + state.data_dir.display(), + state.db_path.display(), + state.master_key.is_some() + ); + + let app = Router::new() + .route("/health", get(health)) + .route("/readyz", get(readyz)) + .route("/v1/files", post(ingest).get(search)) + .route("/v1/files/pending_extract", get(pending_extract)) + .route("/v1/files/claim_extract", post(claim_extract)) + .route("/v1/files/tombstoned", get(tombstoned)) + .route("/v1/files/{file_id}/meta", get(meta)) + .route("/v1/files/{file_id}", get(download)) + .route("/v1/files/{file_id}/link", post(create_link)) + .route("/v1/files/{file_id}/annotations", post(upsert_annotations)) + .route("/v1/files/{file_id}/extract_status", post(set_extract_status)) + .route("/v1/files/{file_id}/tombstone", post(tombstone)) + .route("/v1/public/download", get(public_download)) + .layer(TraceLayer::new_for_http()) + .with_state(state); + + let listener = TcpListener::bind(("0.0.0.0", port)).await?; + axum::serve(listener, app).await?; + Ok(()) +} + diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d3e1adaa24920..ad8d965185f63 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -469,12 +469,86 @@ importers: specifier: workspace:* version: link:../.. + packages/deep-memory-server: + dependencies: + '@hono/node-server': + specifier: ^1.19.0 + version: 1.19.9(hono@4.11.10) + '@qdrant/js-client-rest': + specifier: ^1.15.0 + version: 1.17.0(typescript@5.9.3) + '@xenova/transformers': + specifier: ^2.17.2 + version: 2.17.2 + hono: + specifier: 4.11.10 + version: 4.11.10 + lru-cache: + specifier: ^11.0.2 + version: 11.2.6 + neo4j-driver: + specifier: ^5.28.2 + version: 5.28.3 + pino: + specifier: ^9.6.0 + version: 9.14.0 + prom-client: + specifier: ^15.1.3 + version: 15.1.3 + zod: + specifier: ^4.3.6 + version: 4.3.6 + devDependencies: + '@types/node': + specifier: ^25.2.0 + version: 25.3.1 + tsx: + specifier: ^4.21.0 + version: 4.21.0 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^4.0.18 + version: 4.0.18(@opentelemetry/api@1.9.0)(@types/node@25.3.1)(@vitest/browser-playwright@4.0.18)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2) + packages/moltbot: dependencies: openclaw: specifier: workspace:* version: link:../.. + packages/rustfs-worker: + dependencies: + jszip: + specifier: ^3.10.1 + version: 3.10.1 + mammoth: + specifier: ^1.11.0 + version: 1.11.0 + pdf-parse: + specifier: ^2.4.5 + version: 2.4.5 + pino: + specifier: ^9.6.0 + version: 9.14.0 + zod: + specifier: ^4.3.6 + version: 4.3.6 + devDependencies: + '@types/node': + specifier: ^25.2.0 + version: 25.3.1 + '@types/pdf-parse': + specifier: ^1.1.5 + version: 1.1.5 + tsx: + specifier: ^4.21.0 + version: 4.21.0 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + ui: dependencies: '@lit-labs/signals': @@ -1040,6 +1114,10 @@ packages: peerDependencies: hono: 4.11.10 + '@huggingface/jinja@0.2.2': + resolution: {integrity: sha512-/KPde26khDUIPkTGU82jdtTW9UAuvUTumCAbFs/7giR0SxsvZC4hru51PBvpijH6BVkHcROcvZM/lpy5h1jRRA==} + engines: {node: '>=18'} + '@huggingface/jinja@0.5.5': resolution: {integrity: sha512-xRlzazC+QZwr6z4ixEqYHo9fgwhTZ3xNSdljlKfUFGZSdlvt166DljRELFUfFytlYOYvo3vTisA/AFOuOAzFQQ==} engines: {node: '>=18'} @@ -1444,54 +1522,108 @@ packages: resolution: {integrity: sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==} engines: {node: '>=14.0.0'} + '@napi-rs/canvas-android-arm64@0.1.80': + resolution: {integrity: sha512-sk7xhN/MoXeuExlggf91pNziBxLPVUqF2CAVnB57KLG/pz7+U5TKG8eXdc3pm0d7Od0WreB6ZKLj37sX9muGOQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + '@napi-rs/canvas-android-arm64@0.1.95': resolution: {integrity: sha512-SqTh0wsYbetckMXEvHqmR7HKRJujVf1sYv1xdlhkifg6TlCSysz1opa49LlS3+xWuazcQcfRfmhA07HxxxGsAA==} engines: {node: '>= 10'} cpu: [arm64] os: [android] + '@napi-rs/canvas-darwin-arm64@0.1.80': + resolution: {integrity: sha512-O64APRTXRUiAz0P8gErkfEr3lipLJgM6pjATwavZ22ebhjYl/SUbpgM0xcWPQBNMP1n29afAC/Us5PX1vg+JNQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + '@napi-rs/canvas-darwin-arm64@0.1.95': resolution: {integrity: sha512-F7jT0Syu+B9DGBUBcMk3qCRIxAWiDXmvEjamwbYfbZl7asI1pmXZUnCOoIu49Wt0RNooToYfRDxU9omD6t5Xuw==} engines: {node: '>= 10'} cpu: [arm64] os: [darwin] + '@napi-rs/canvas-darwin-x64@0.1.80': + resolution: {integrity: sha512-FqqSU7qFce0Cp3pwnTjVkKjjOtxMqRe6lmINxpIZYaZNnVI0H5FtsaraZJ36SiTHNjZlUB69/HhxNDT1Aaa9vA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + '@napi-rs/canvas-darwin-x64@0.1.95': resolution: {integrity: sha512-54eb2Ho15RDjYGXO/harjRznBrAvu+j5nQ85Z4Qd6Qg3slR8/Ja+Yvvy9G4yo7rdX6NR9GPkZeSTf2UcKXwaXw==} engines: {node: '>= 10'} cpu: [x64] os: [darwin] + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.80': + resolution: {integrity: sha512-eyWz0ddBDQc7/JbAtY4OtZ5SpK8tR4JsCYEZjCE3dI8pqoWUC8oMwYSBGCYfsx2w47cQgQCgMVRVTFiiO38hHQ==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.95': resolution: {integrity: sha512-hYaLCSLx5bmbnclzQc3ado3PgZ66blJWzjXp0wJmdwpr/kH+Mwhj6vuytJIomgksyJoCdIqIa4N6aiqBGJtJ5Q==} engines: {node: '>= 10'} cpu: [arm] os: [linux] + '@napi-rs/canvas-linux-arm64-gnu@0.1.80': + resolution: {integrity: sha512-qwA63t8A86bnxhuA/GwOkK3jvb+XTQaTiVML0vAWoHyoZYTjNs7BzoOONDgTnNtr8/yHrq64XXzUoLqDzU+Uuw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + '@napi-rs/canvas-linux-arm64-gnu@0.1.95': resolution: {integrity: sha512-J7VipONahKsmScPZsipHVQBqpbZx4favaD8/enWzzlGcjiwycOoymL7f4tNeqdjK0su19bDOUt6mjp9gsPWYlw==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] + '@napi-rs/canvas-linux-arm64-musl@0.1.80': + resolution: {integrity: sha512-1XbCOz/ymhj24lFaIXtWnwv/6eFHXDrjP0jYkc6iHQ9q8oXKzUX1Lc6bu+wuGiLhGh2GS/2JlfORC5ZcXimRcg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + '@napi-rs/canvas-linux-arm64-musl@0.1.95': resolution: {integrity: sha512-PXy0UT1J/8MPG8UAkWp6Fd51ZtIZINFzIjGH909JjQrtCuJf3X6nanHYdz1A+Wq9o4aoPAw1YEUpFS1lelsVlg==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] + '@napi-rs/canvas-linux-riscv64-gnu@0.1.80': + resolution: {integrity: sha512-XTzR125w5ZMs0lJcxRlS1K3P5RaZ9RmUsPtd1uGt+EfDyYMu4c6SEROYsxyatbbu/2+lPe7MPHOO/0a0x7L/gw==} + engines: {node: '>= 10'} + cpu: [riscv64] + os: [linux] + '@napi-rs/canvas-linux-riscv64-gnu@0.1.95': resolution: {integrity: sha512-2IzCkW2RHRdcgF9W5/plHvYFpc6uikyjMb5SxjqmNxfyDFz9/HB89yhi8YQo0SNqrGRI7yBVDec7Pt+uMyRWsg==} engines: {node: '>= 10'} cpu: [riscv64] os: [linux] + '@napi-rs/canvas-linux-x64-gnu@0.1.80': + resolution: {integrity: sha512-BeXAmhKg1kX3UCrJsYbdQd3hIMDH/K6HnP/pG2LuITaXhXBiNdh//TVVVVCBbJzVQaV5gK/4ZOCMrQW9mvuTqA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + '@napi-rs/canvas-linux-x64-gnu@0.1.95': resolution: {integrity: sha512-OV/ol/OtcUr4qDhQg8G7SdViZX8XyQeKpPsVv/j3+7U178FGoU4M+yIocdVo1ih/A8GQ63+LjF4jDoEjaVU8Pw==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + '@napi-rs/canvas-linux-x64-musl@0.1.80': + resolution: {integrity: sha512-x0XvZWdHbkgdgucJsRxprX/4o4sEed7qo9rCQA9ugiS9qE2QvP0RIiEugtZhfLH3cyI+jIRFJHV4Fuz+1BHHMg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + '@napi-rs/canvas-linux-x64-musl@0.1.95': resolution: {integrity: sha512-Z5KzqBK/XzPz5+SFHKz7yKqClEQ8pOiEDdgk5SlphBLVNb8JFIJkxhtJKSvnJyHh2rjVgiFmvtJzMF0gNwwKyQ==} engines: {node: '>= 10'} @@ -1504,12 +1636,22 @@ packages: cpu: [arm64] os: [win32] + '@napi-rs/canvas-win32-x64-msvc@0.1.80': + resolution: {integrity: sha512-Z8jPsM6df5V8B1HrCHB05+bDiCxjE9QA//3YrkKIdVDEwn5RKaqOxCJDRJkl48cJbylcrJbW4HxZbTte8juuPg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + '@napi-rs/canvas-win32-x64-msvc@0.1.95': resolution: {integrity: sha512-GA8leTTCfdjuHi8reICTIxU0081PhXvl3lzIniLUjeLACx9GubUiyzkwFb+oyeKLS5IAGZFLKnzAf4wm2epRlA==} engines: {node: '>= 10'} cpu: [x64] os: [win32] + '@napi-rs/canvas@0.1.80': + resolution: {integrity: sha512-DxuT1ClnIPts1kQx8FBmkk4BQDTfI5kIzywAaMjQSXfNnra5UFU9PwurXrl+Je3bJ6BGsp/zmshVVFbCmyI+ww==} + engines: {node: '>= 10'} + '@napi-rs/canvas@0.1.95': resolution: {integrity: sha512-lkg23ge+rgyhgUwXmlbkPEhuhHq/hUi/gXKH+4I7vO+lJrbNfEYcQdJLIGjKyXLQzgFiiyDAwh5vAe/tITAE+w==} engines: {node: '>= 10'} @@ -2190,6 +2332,16 @@ packages: '@protobufjs/utf8@1.1.0': resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + '@qdrant/js-client-rest@1.17.0': + resolution: {integrity: sha512-aZFQeirWVqWAa1a8vJ957LMzcXkFHGbsoRhzc8AkGfg6V0jtK8PlG8/eyyc2xhYsR961FDDx1Tx6nyE0K7lS+A==} + engines: {node: '>=18.17.0', pnpm: '>=8'} + peerDependencies: + typescript: '>=4.7' + + '@qdrant/openapi-typescript-fetch@1.2.6': + resolution: {integrity: sha512-oQG/FejNpItrxRHoyctYvT3rwGZOnK4jr3JdppO/c78ktDvkWiPXPHNsrDf33K9sZdRb6PR7gi4noIapu5q4HA==} + engines: {node: '>=18.0.0', pnpm: '>=8'} + '@quansync/fs@1.0.0': resolution: {integrity: sha512-4TJ3DFtlf1L5LDMaM6CanJ/0lckGNtJcMjQ1NAV6zDmA0tEHKZtxNKin8EgPaVX1YzljbxckyT2tJrpQKAtngQ==} @@ -2907,6 +3059,9 @@ packages: '@types/node@25.3.1': resolution: {integrity: sha512-hj9YIJimBCipHVfHKRMnvmHg+wfhKc0o4mTtXh9pKBjC8TLJzz0nzGmLi5UJsYAUgSvXFHgb0V2oY10DUFtImw==} + '@types/pdf-parse@1.1.5': + resolution: {integrity: sha512-kBfrSXsloMnUJOKi25s3+hRmkycHfLK6A09eRGqF/N8BkQoPUmaCr+q8Cli5FnfohEz/rsv82zAiPz/LXtOGhA==} + '@types/qrcode-terminal@0.12.2': resolution: {integrity: sha512-v+RcIEJ+Uhd6ygSQ0u5YYY7ZM+la7GgPbs0V/7l/kFs2uO4S8BcIUEMoP7za4DNIqNnUD5npf0A/7kBhrCKG5Q==} @@ -3078,6 +3233,13 @@ packages: resolution: {tarball: https://codeload.github.com/whiskeysockets/libsignal-node/tar.gz/1c30d7d7e76a3b0aa120b04dc6a26f5a12dccf67} version: 2.0.1 + '@xenova/transformers@2.17.2': + resolution: {integrity: sha512-lZmHqzrVIkSvZdKZEx7IYY51TK0WDrC8eR0c5IMnBsO8di8are1zzw8BlLhyO2TklZKLN5UffNGs1IJwT6oOqQ==} + + '@xmldom/xmldom@0.8.11': + resolution: {integrity: sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw==} + engines: {node: '>=10.0.0'} + abbrev@1.1.1: resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==} @@ -3169,6 +3331,9 @@ packages: engines: {node: '>=10'} deprecated: This package is no longer supported. + argparse@1.0.10: + resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} + argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} @@ -3260,6 +3425,36 @@ packages: bare-abort-controller: optional: true + bare-fs@4.5.5: + resolution: {integrity: sha512-XvwYM6VZqKoqDll8BmSww5luA5eflDzY0uEFfBJtFKe4PAAtxBjU3YIxzIBzhyaEQBy1VXEQBto4cpN5RZJw+w==} + engines: {bare: '>=1.16.0'} + peerDependencies: + bare-buffer: '*' + peerDependenciesMeta: + bare-buffer: + optional: true + + bare-os@3.7.0: + resolution: {integrity: sha512-64Rcwj8qlnTZU8Ps6JJEdSmxBEUGgI7g8l+lMtsJLl4IsfTcHMTfJ188u2iGV6P6YPRZrtv72B2kjn+hp+Yv3g==} + engines: {bare: '>=1.14.0'} + + bare-path@3.0.0: + resolution: {integrity: sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==} + + bare-stream@2.8.0: + resolution: {integrity: sha512-reUN0M2sHRqCdG4lUK3Fw8w98eeUIZHL5c3H7Mbhk2yVBL+oofgaIp0ieLfD5QXwPCypBpmEEKU2WZKzbAk8GA==} + peerDependencies: + bare-buffer: '*' + bare-events: '*' + peerDependenciesMeta: + bare-buffer: + optional: true + bare-events: + optional: true + + bare-url@2.3.2: + resolution: {integrity: sha512-ZMq4gd9ngV5aTMa5p9+UfY0b3skwhHELaDkhEHetMdX0LRkW9kzaym4oo/Eh+Ghm0CCDuMTsRIGM/ytUc1ZYmw==} + base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} @@ -3280,9 +3475,18 @@ packages: bignumber.js@9.3.1: resolution: {integrity: sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==} + bintrees@1.0.2: + resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} + birpc@4.0.0: resolution: {integrity: sha512-LShSxJP0KTmd101b6DRyGBj57LZxSDYWKitQNW/mi8GRMvZb078Uf9+pveax1DrVL89vm7mWe+TovdI/UDOuPw==} + bl@4.1.0: + resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} + + bluebird@3.4.7: + resolution: {integrity: sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==} + bluebird@3.7.2: resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==} @@ -3316,6 +3520,12 @@ packages: buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + buffer@5.7.1: + resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + + buffer@6.0.3: + resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + bun-types@1.3.9: resolution: {integrity: sha512-+UBWWOakIP4Tswh0Bt0QD0alpTY8cb5hvgiYeWCMet9YukHbzuruIEeXC2D7nMJPB12kbh8C7XJykSexEqGKJg==} @@ -3364,6 +3574,9 @@ packages: resolution: {integrity: sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==} engines: {node: '>= 20.19.0'} + chownr@1.1.4: + resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} + chownr@3.0.0: resolution: {integrity: sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==} engines: {node: '>=18'} @@ -3414,10 +3627,17 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + color-string@1.9.1: + resolution: {integrity: sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==} + color-support@1.1.3: resolution: {integrity: sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==} hasBin: true + color@4.2.3: + resolution: {integrity: sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==} + engines: {node: '>=12.5.0'} + combined-stream@1.0.8: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} @@ -3524,6 +3744,10 @@ packages: supports-color: optional: true + decompress-response@6.0.0: + resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} + engines: {node: '>=10'} + deep-extend@0.6.0: resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} engines: {node: '>=4.0.0'} @@ -3562,6 +3786,9 @@ packages: resolution: {integrity: sha512-qejHi7bcSD4hQAZE0tNAawRK1ZtafHDmMTMkrrIGgSLl7hTnQHmKCeB45xAcbfTqK2zowkM3j3bHt/4b/ARbYQ==} engines: {node: '>=0.3.1'} + dingbat-to-unicode@1.0.1: + resolution: {integrity: sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w==} + discord-api-types@0.38.37: resolution: {integrity: sha512-Cv47jzY1jkGkh5sv0bfHYqGgKOWO1peOrGMkDFM4UmaGMOTgOW8QSexhvixa9sVOiz8MnVOBryWYyw/CEVhj7w==} @@ -3597,6 +3824,9 @@ packages: oxc-resolver: optional: true + duck@0.1.12: + resolution: {integrity: sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg==} + dunder-proto@1.0.1: resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} engines: {node: '>= 0.4'} @@ -3718,6 +3948,10 @@ packages: events-universal@1.0.1: resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==} + expand-template@2.0.3: + resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} + engines: {node: '>=6'} + expect-type@1.3.0: resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} engines: {node: '>=12.0.0'} @@ -3798,6 +4032,9 @@ packages: resolution: {integrity: sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==} engines: {node: '>=4.0.0'} + flatbuffers@1.12.0: + resolution: {integrity: sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==} + flatbuffers@24.12.23: resolution: {integrity: sha512-dLVCAISd5mhls514keQzmEG6QHmUUsNuWsb4tFafIUwvvgDjXhtfAYSKOzt5SWOy+qByV5pbsDZ+Vb7HUOBEdA==} @@ -3838,6 +4075,9 @@ packages: resolution: {integrity: sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==} engines: {node: '>= 0.8'} + fs-constants@1.0.0: + resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} + fs-extra@11.3.3: resolution: {integrity: sha512-VWSRii4t0AFm6ixFFmLLx1t7wS1gh+ckoa84aOeapGum0h+EZd1EhEumSB+ZdDLnEPuucsVB9oB7cxJHap6Afg==} engines: {node: '>=14.14'} @@ -3905,6 +4145,9 @@ packages: getpass@0.1.7: resolution: {integrity: sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==} + github-from-package@0.0.0: + resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} + glob-to-regexp@0.4.1: resolution: {integrity: sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==} @@ -3948,6 +4191,9 @@ packages: resolution: {integrity: sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw==} engines: {node: '>=18'} + guid-typescript@1.0.9: + resolution: {integrity: sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==} + has-flag@4.0.0: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} @@ -4089,6 +4335,9 @@ packages: ircv3@0.33.0: resolution: {integrity: sha512-7rK1Aial3LBiFycE8w3MHiBBFb41/2GG2Ll/fR2IJj1vx0pLpn1s+78K+z/I4PZTqCCSp/Sb4QgKMh3NMhx0Kg==} + is-arrayish@0.3.4: + resolution: {integrity: sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA==} + is-electron@2.2.2: resolution: {integrity: sha512-FO/Rhvz5tuw4MCWkpMzHFKWD2LsfHzIb7i6MdPYZ/KW7AlxawyLkqdy+jPZP1WubqEADE3O4FUENlJHDfQASRg==} @@ -4391,6 +4640,9 @@ packages: long@5.3.2: resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + lop@0.4.2: + resolution: {integrity: sha512-RefILVDQ4DKoRZsJ4Pj22TxE3omDO47yFpkIBoDKzkqPRISs5U1cnAdg/5583YPkWPaLIYHOKRMQSvjFsO26cw==} + lowdb@1.0.0: resolution: {integrity: sha512-2+x8esE/Wb9SQ1F9IHaYWfsC9FIecLOPrK4g17FGEayjUWH172H6nwicRovGvSE2CPZouc2MCIqCI7h9d+GftQ==} engines: {node: '>=4'} @@ -4431,6 +4683,11 @@ packages: resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} engines: {node: '>=10'} + mammoth@1.11.0: + resolution: {integrity: sha512-BcEqqY/BOwIcI1iR5tqyVlqc3KIaMRa4egSoK83YAVrBf6+yqdAAbtUcFDCWX8Zef8/fgNZ6rl4VUv+vVX8ddQ==} + engines: {node: '>=12.0.0'} + hasBin: true + markdown-it@14.1.1: resolution: {integrity: sha512-BuU2qnTti9YKgK5N+IeMubp14ZUKUUw7yeJbkjtosvHiP0AZ5c8IAgEMk79D0eC8F23r4Ac/q8cAIFdm2FtyoA==} hasBin: true @@ -4496,6 +4753,10 @@ packages: resolution: {integrity: sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==} engines: {node: '>=18'} + mimic-response@3.1.0: + resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} + engines: {node: '>=10'} + minimalistic-assert@1.0.1: resolution: {integrity: sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==} @@ -4514,6 +4775,9 @@ packages: resolution: {integrity: sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==} engines: {node: '>= 18'} + mkdirp-classic@0.5.3: + resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} + mkdirp@3.0.1: resolution: {integrity: sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==} engines: {node: '>=10'} @@ -4556,6 +4820,9 @@ packages: engines: {node: ^18 || >=20} hasBin: true + napi-build-utils@2.0.0: + resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==} + negotiator@0.6.3: resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} engines: {node: '>= 0.6'} @@ -4564,10 +4831,26 @@ packages: resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==} engines: {node: '>= 0.6'} + neo4j-driver-bolt-connection@5.28.3: + resolution: {integrity: sha512-wqHBYcU0FVRDmdsoZ+Fk0S/InYmu9/4BT6fPYh45Jimg/J7vQBUcdkiHGU7nop7HRb1ZgJmL305mJb6g5Bv35Q==} + + neo4j-driver-core@5.28.3: + resolution: {integrity: sha512-Jk+hAmjFmO5YzVH/U7FyKXigot9zmIfLz6SZQy0xfr4zfTE/S8fOYFOGqKQTHBE86HHOWH2RbTslbxIb+XtU2g==} + + neo4j-driver@5.28.3: + resolution: {integrity: sha512-k7c0wEh3HoONv1v5AyLp9/BDAbYHJhz2TZvzWstSEU3g3suQcXmKEaYBfrK2UMzxcy3bCT0DrnfRbzsOW5G/Ag==} + netmask@2.0.2: resolution: {integrity: sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==} engines: {node: '>= 0.4.0'} + node-abi@3.87.0: + resolution: {integrity: sha512-+CGM1L1CgmtheLcBuleyYOn7NWPVu0s0EJH2C4puxgEZb9h8QpR9G2dBfZJOAUhi7VQxuBPMd0hiISWcTyiYyQ==} + engines: {node: '>=10'} + + node-addon-api@6.1.0: + resolution: {integrity: sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==} + node-addon-api@8.5.0: resolution: {integrity: sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A==} engines: {node: ^18 || ^20 || >= 21} @@ -4682,6 +4965,19 @@ packages: resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==} engines: {node: '>=18'} + onnx-proto@4.0.4: + resolution: {integrity: sha512-aldMOB3HRoo6q/phyB6QRQxSt895HNNw82BNyZ2CMh4bjeKv7g/c+VpAFtJuEMVfYLMbRx61hbuqnKceLeDcDA==} + + onnxruntime-common@1.14.0: + resolution: {integrity: sha512-3LJpegM2iMNRX2wUmtYfeX/ytfOzNwAWKSq1HbRrKc9+uqG/FsEA0bbKZl1btQeZaXhC26l44NWpNUeXPII7Ew==} + + onnxruntime-node@1.14.0: + resolution: {integrity: sha512-5ba7TWomIV/9b6NH/1x/8QEeowsb+jBEvFzU6z0T4mNsFwdPqXeFUM7uxC6QeSRkEbWu3qEB0VMjrvzN/0S9+w==} + os: [win32, darwin, linux] + + onnxruntime-web@1.14.0: + resolution: {integrity: sha512-Kcqf43UMfW8mCydVGcX9OMXI2VN17c0p6XvR7IPSZzBf/6lteBzXHvcEVWDPmCKuGombl997HgLqj91F11DzXw==} + openai@6.10.0: resolution: {integrity: sha512-ITxOGo7rO3XRMiKA5l7tQ43iNNu+iXGFAcf2t+aWVzzqRaS0i7m1K2BhxNdaveB+5eENhO0VY1FkiZzhBk4v3A==} hasBin: true @@ -4714,6 +5010,9 @@ packages: '@napi-rs/canvas': ^0.1.89 node-llama-cpp: 3.15.1 + option@0.2.4: + resolution: {integrity: sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==} + opus-decoder@0.7.11: resolution: {integrity: sha512-+e+Jz3vGQLxRTBHs8YJQPRPc1Tr+/aC6coV/DlZylriA29BdHQAYXhvNRKtjftof17OFng0+P4wsFIqQu3a48A==} @@ -4840,6 +5139,15 @@ packages: pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + pdf-parse@2.4.5: + resolution: {integrity: sha512-mHU89HGh7v+4u2ubfnevJ03lmPgQ5WU4CxAVmTSh/sxVTEDYd1er/dKS/A6vg77NX47KTEoihq8jZBLr8Cxuwg==} + engines: {node: '>=20.16.0 <21 || >=22.3.0'} + hasBin: true + + pdfjs-dist@5.4.296: + resolution: {integrity: sha512-DlOzet0HO7OEnmUmB6wWGJrrdvbyJKftI1bhMitK7O2N8W2gc757yyYBbINy9IDafXAV9wmKr9t7xsTaNKRG5Q==} + engines: {node: '>=20.16.0 || >=22.3.0'} + pdfjs-dist@5.4.624: resolution: {integrity: sha512-sm6TxKTtWv1Oh6n3C6J6a8odejb5uO4A4zo/2dgkHuC0iu8ZMAXOezEODkVaoVp8nX1Xzr+0WxFJJmUr45hQzg==} engines: {node: '>=20.16.0 || >=22.3.0'} @@ -4878,6 +5186,9 @@ packages: resolution: {integrity: sha512-1wrVzJ2STrpmONHKBy228LM1b84msXDUoAzVEl0R8Mz4Ce6EPr+IVtxm8+yvrqLYMHswREkjYFaMxnyGnaY3Ng==} hasBin: true + platform@1.3.6: + resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==} + playwright-core@1.58.2: resolution: {integrity: sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==} engines: {node: '>=18'} @@ -4900,6 +5211,12 @@ packages: resolution: {integrity: sha512-d+JFcLM17njZaOLkv6SCev7uoLaBtfK86vMUXhW1Z4glPWh4jozno9APvW/XKFJ3CCxVoC7OL38BqRydtu5nGg==} engines: {node: '>=12'} + prebuild-install@7.1.3: + resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} + engines: {node: '>=10'} + deprecated: No longer maintained. Please contact the author of the relevant native addon; alternatives are available. + hasBin: true + pretty-bytes@6.1.1: resolution: {integrity: sha512-mQUvGU6aUFQ+rNvTIAcZuWGRT9a6f6Yrg9bHs4ImKF+HZCEK+plBvnAZYSIQztknZF2qnzNtr6F8s0+IuptdlQ==} engines: {node: ^14.13.1 || >=16.0.0} @@ -4935,6 +5252,10 @@ packages: process-warning@5.0.0: resolution: {integrity: sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==} + prom-client@15.1.3: + resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==} + engines: {node: ^16 || ^18 || >=20} + proper-lockfile@4.1.2: resolution: {integrity: sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==} @@ -5111,6 +5432,9 @@ packages: resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==} engines: {node: '>= 18'} + rxjs@7.8.2: + resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==} + safe-buffer@5.1.2: resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} @@ -5164,6 +5488,10 @@ packages: setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + sharp@0.32.6: + resolution: {integrity: sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w==} + engines: {node: '>=14.15.0'} + sharp@0.34.5: resolution: {integrity: sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} @@ -5210,9 +5538,18 @@ packages: peerDependencies: signal-polyfill: ^0.2.0 + simple-concat@1.0.1: + resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} + + simple-get@4.0.1: + resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} + simple-git@3.32.3: resolution: {integrity: sha512-56a5oxFdWlsGygOXHWrG+xjj5w9ZIt2uQbzqiIGdR/6i5iococ7WQ/bNPzWxCJdEUGUCmyMH0t9zMpRJTaKxmw==} + simple-swizzle@0.2.4: + resolution: {integrity: sha512-nAu1WFPQSMNr2Zn9PGSZK9AGn4t/y97lEm+MXTtUDwfP0ksAIX4nO+6ruD9Jwut4C49SB1Ws+fbXsm/yScWOHw==} + simple-yenc@1.0.4: resolution: {integrity: sha512-5gvxpSd79e9a3V4QDYUqnqxeD4HGlhCakVpb6gMnDD7lexJggSBJRBO5h52y/iJrdXRilX9UCuDaIJhSWm5OWw==} @@ -5269,6 +5606,9 @@ packages: resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} engines: {node: '>= 10.x'} + sprintf-js@1.0.3: + resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + sqlite-vec-darwin-arm64@0.1.7-alpha.2: resolution: {integrity: sha512-raIATOqFYkeCHhb/t3r7W7Cf2lVYdf4J3ogJ6GFc8PQEgHCPEsi+bYnm2JT84MzLfTlSTIdxr4/NKv+zF7oLPw==} cpu: [arm64] @@ -5383,6 +5723,16 @@ packages: resolution: {integrity: sha512-iK5/YhZxq5GO5z8wb0bY1317uDF3Zjpha0QFFLA8/trAoiLbQD0HUbMesEaxyzUgDxi2QlcbM8IvqOlEjgoXBA==} engines: {node: '>=12.17'} + tar-fs@2.1.4: + resolution: {integrity: sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==} + + tar-fs@3.1.1: + resolution: {integrity: sha512-LZA0oaPOc2fVo82Txf3gw+AkEd38szODlptMYejQUhndHMLQ9M059uXR+AfS7DNo0NpINvSqDsvyaCrBVkptWg==} + + tar-stream@2.2.0: + resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} + engines: {node: '>=6'} + tar-stream@3.1.7: resolution: {integrity: sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==} @@ -5391,6 +5741,12 @@ packages: engines: {node: '>=18'} deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me + tdigest@0.1.2: + resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} + + teex@1.0.1: + resolution: {integrity: sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==} + text-decoder@1.2.7: resolution: {integrity: sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==} @@ -5534,6 +5890,9 @@ packages: unconfig-core@7.5.0: resolution: {integrity: sha512-Su3FauozOGP44ZmKdHy2oE6LPjk51M/TRRjHv2HNCWiDvfvCoxC2lno6jevMA91MYAdCdwP05QnWdWpSbncX/w==} + underscore@1.13.8: + resolution: {integrity: sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==} + undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} @@ -5543,6 +5902,10 @@ packages: undici-types@7.18.2: resolution: {integrity: sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==} + undici@6.23.0: + resolution: {integrity: sha512-VfQPToRA5FZs/qJxLIinmU59u0r7LXqoJkCzinq3ckNJp3vKEh7jTWN589YQ5+aoAC/TGRLyJLCPKcLQbM8r9g==} + engines: {node: '>=18.17'} + undici@7.22.0: resolution: {integrity: sha512-RqslV2Us5BrllB+JeiZnK4peryVTndy9Dnqq62S3yYRRTj0tFQCwEniUy2167skdGOy3vqRzEvl1Dm4sV2ReDg==} engines: {node: '>=20.18.1'} @@ -5740,6 +6103,10 @@ packages: utf-8-validate: optional: true + xmlbuilder@10.1.1: + resolution: {integrity: sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==} + engines: {node: '>=4.0'} + y18n@5.0.8: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} @@ -6625,7 +6992,8 @@ snapshots: '@hono/node-server@1.19.9(hono@4.11.10)': dependencies: hono: 4.11.10 - optional: true + + '@huggingface/jinja@0.2.2': {} '@huggingface/jinja@0.5.5': {} @@ -7102,39 +7470,82 @@ snapshots: '@mozilla/readability@0.6.0': {} + '@napi-rs/canvas-android-arm64@0.1.80': + optional: true + '@napi-rs/canvas-android-arm64@0.1.95': optional: true + '@napi-rs/canvas-darwin-arm64@0.1.80': + optional: true + '@napi-rs/canvas-darwin-arm64@0.1.95': optional: true + '@napi-rs/canvas-darwin-x64@0.1.80': + optional: true + '@napi-rs/canvas-darwin-x64@0.1.95': optional: true + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.80': + optional: true + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.95': optional: true + '@napi-rs/canvas-linux-arm64-gnu@0.1.80': + optional: true + '@napi-rs/canvas-linux-arm64-gnu@0.1.95': optional: true + '@napi-rs/canvas-linux-arm64-musl@0.1.80': + optional: true + '@napi-rs/canvas-linux-arm64-musl@0.1.95': optional: true + '@napi-rs/canvas-linux-riscv64-gnu@0.1.80': + optional: true + '@napi-rs/canvas-linux-riscv64-gnu@0.1.95': optional: true + '@napi-rs/canvas-linux-x64-gnu@0.1.80': + optional: true + '@napi-rs/canvas-linux-x64-gnu@0.1.95': optional: true + '@napi-rs/canvas-linux-x64-musl@0.1.80': + optional: true + '@napi-rs/canvas-linux-x64-musl@0.1.95': optional: true '@napi-rs/canvas-win32-arm64-msvc@0.1.95': optional: true + '@napi-rs/canvas-win32-x64-msvc@0.1.80': + optional: true + '@napi-rs/canvas-win32-x64-msvc@0.1.95': optional: true + '@napi-rs/canvas@0.1.80': + optionalDependencies: + '@napi-rs/canvas-android-arm64': 0.1.80 + '@napi-rs/canvas-darwin-arm64': 0.1.80 + '@napi-rs/canvas-darwin-x64': 0.1.80 + '@napi-rs/canvas-linux-arm-gnueabihf': 0.1.80 + '@napi-rs/canvas-linux-arm64-gnu': 0.1.80 + '@napi-rs/canvas-linux-arm64-musl': 0.1.80 + '@napi-rs/canvas-linux-riscv64-gnu': 0.1.80 + '@napi-rs/canvas-linux-x64-gnu': 0.1.80 + '@napi-rs/canvas-linux-x64-musl': 0.1.80 + '@napi-rs/canvas-win32-x64-msvc': 0.1.80 + '@napi-rs/canvas@0.1.95': optionalDependencies: '@napi-rs/canvas-android-arm64': 0.1.95 @@ -7755,6 +8166,14 @@ snapshots: '@protobufjs/utf8@1.1.0': {} + '@qdrant/js-client-rest@1.17.0(typescript@5.9.3)': + dependencies: + '@qdrant/openapi-typescript-fetch': 1.2.6 + typescript: 5.9.3 + undici: 6.23.0 + + '@qdrant/openapi-typescript-fetch@1.2.6': {} + '@quansync/fs@1.0.0': dependencies: quansync: 1.0.0 @@ -8546,6 +8965,10 @@ snapshots: dependencies: undici-types: 7.18.2 + '@types/pdf-parse@1.1.5': + dependencies: + '@types/node': 25.3.1 + '@types/qrcode-terminal@0.12.2': {} '@types/qs@6.14.0': {} @@ -8793,6 +9216,20 @@ snapshots: curve25519-js: 0.0.4 protobufjs: 6.8.8 + '@xenova/transformers@2.17.2': + dependencies: + '@huggingface/jinja': 0.2.2 + onnxruntime-web: 1.14.0 + sharp: 0.32.6 + optionalDependencies: + onnxruntime-node: 1.14.0 + transitivePeerDependencies: + - bare-abort-controller + - bare-buffer + - react-native-b4a + + '@xmldom/xmldom@0.8.11': {} + abbrev@1.1.1: optional: true @@ -8885,6 +9322,10 @@ snapshots: readable-stream: 3.6.2 optional: true + argparse@1.0.10: + dependencies: + sprintf-js: 1.0.3 + argparse@2.0.1: {} array-back@3.1.0: {} @@ -8967,6 +9408,42 @@ snapshots: bare-events@2.8.2: {} + bare-fs@4.5.5: + dependencies: + bare-events: 2.8.2 + bare-path: 3.0.0 + bare-stream: 2.8.0(bare-events@2.8.2) + bare-url: 2.3.2 + fast-fifo: 1.3.2 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + optional: true + + bare-os@3.7.0: + optional: true + + bare-path@3.0.0: + dependencies: + bare-os: 3.7.0 + optional: true + + bare-stream@2.8.0(bare-events@2.8.2): + dependencies: + streamx: 2.23.0 + teex: 1.0.1 + optionalDependencies: + bare-events: 2.8.2 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + optional: true + + bare-url@2.3.2: + dependencies: + bare-path: 3.0.0 + optional: true + base64-js@1.5.1: {} basic-auth@2.0.1: @@ -8983,8 +9460,18 @@ snapshots: bignumber.js@9.3.1: {} + bintrees@1.0.2: {} + birpc@4.0.0: {} + bl@4.1.0: + dependencies: + buffer: 5.7.1 + inherits: 2.0.4 + readable-stream: 3.6.2 + + bluebird@3.4.7: {} + bluebird@3.7.2: {} body-parser@1.20.4: @@ -9034,6 +9521,16 @@ snapshots: buffer-from@1.1.2: {} + buffer@5.7.1: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + + buffer@6.0.3: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + bun-types@1.3.9: dependencies: '@types/node': 25.3.1 @@ -9082,6 +9579,8 @@ snapshots: dependencies: readdirp: 5.0.0 + chownr@1.1.4: {} + chownr@3.0.0: {} ci-info@4.4.0: {} @@ -9140,9 +9639,19 @@ snapshots: color-name@1.1.4: {} + color-string@1.9.1: + dependencies: + color-name: 1.1.4 + simple-swizzle: 0.2.4 + color-support@1.1.3: optional: true + color@4.2.3: + dependencies: + color-convert: 2.0.1 + color-string: 1.9.1 + combined-stream@1.0.8: dependencies: delayed-stream: 1.0.0 @@ -9226,6 +9735,10 @@ snapshots: dependencies: ms: 2.1.3 + decompress-response@6.0.0: + dependencies: + mimic-response: 3.1.0 + deep-extend@0.6.0: {} deepmerge@4.3.1: {} @@ -9251,6 +9764,8 @@ snapshots: diff@8.0.3: {} + dingbat-to-unicode@1.0.1: {} + discord-api-types@0.38.37: {} discord-api-types@0.38.40: {} @@ -9281,6 +9796,10 @@ snapshots: dts-resolver@2.1.3: {} + duck@0.1.12: + dependencies: + underscore: 1.13.8 + dunder-proto@1.0.1: dependencies: call-bind-apply-helpers: 1.0.2 @@ -9404,6 +9923,8 @@ snapshots: transitivePeerDependencies: - bare-abort-controller + expand-template@2.0.3: {} + expect-type@1.3.0: {} express@4.22.1: @@ -9556,6 +10077,8 @@ snapshots: dependencies: array-back: 3.1.0 + flatbuffers@1.12.0: {} + flatbuffers@24.12.23: {} follow-redirects@1.15.11: {} @@ -9586,6 +10109,8 @@ snapshots: fresh@2.0.0: {} + fs-constants@1.0.0: {} + fs-extra@11.3.3: dependencies: graceful-fs: 4.2.11 @@ -9683,6 +10208,8 @@ snapshots: dependencies: assert-plus: 1.0.0 + github-from-package@0.0.0: {} + glob-to-regexp@0.4.1: {} glob@10.5.0: @@ -9756,6 +10283,8 @@ snapshots: transitivePeerDependencies: - supports-color + guid-typescript@1.0.9: {} + has-flag@4.0.0: {} has-own@1.0.1: {} @@ -9784,8 +10313,7 @@ snapshots: highlight.js@10.7.3: {} - hono@4.11.10: - optional: true + hono@4.11.10: {} hookable@6.0.1: {} @@ -9935,6 +10463,8 @@ snapshots: - bufferutil - utf-8-validate + is-arrayish@0.3.4: {} + is-electron@2.2.2: {} is-fullwidth-code-point@3.0.0: {} @@ -10204,6 +10734,12 @@ snapshots: long@5.3.2: {} + lop@0.4.2: + dependencies: + duck: 0.1.12 + option: 0.2.4 + underscore: 1.13.8 + lowdb@1.0.0: dependencies: graceful-fs: 4.2.11 @@ -10250,6 +10786,19 @@ snapshots: dependencies: semver: 7.7.4 + mammoth@1.11.0: + dependencies: + '@xmldom/xmldom': 0.8.11 + argparse: 1.0.10 + base64-js: 1.5.1 + bluebird: 3.4.7 + dingbat-to-unicode: 1.0.1 + jszip: 3.10.1 + lop: 0.4.2 + path-is-absolute: 1.0.1 + underscore: 1.13.8 + xmlbuilder: 10.1.1 + markdown-it@14.1.1: dependencies: argparse: 2.0.1 @@ -10293,6 +10842,8 @@ snapshots: mimic-function@5.0.1: {} + mimic-response@3.1.0: {} + minimalistic-assert@1.0.1: {} minimatch@10.2.4: @@ -10307,6 +10858,8 @@ snapshots: dependencies: minipass: 7.1.3 + mkdirp-classic@0.5.3: {} + mkdirp@3.0.1: {} module-details-from-path@1.0.4: {} @@ -10357,12 +10910,34 @@ snapshots: nanoid@5.1.6: {} + napi-build-utils@2.0.0: {} + negotiator@0.6.3: {} negotiator@1.0.0: {} + neo4j-driver-bolt-connection@5.28.3: + dependencies: + buffer: 6.0.3 + neo4j-driver-core: 5.28.3 + string_decoder: 1.3.0 + + neo4j-driver-core@5.28.3: {} + + neo4j-driver@5.28.3: + dependencies: + neo4j-driver-bolt-connection: 5.28.3 + neo4j-driver-core: 5.28.3 + rxjs: 7.8.2 + netmask@2.0.2: {} + node-abi@3.87.0: + dependencies: + semver: 7.7.4 + + node-addon-api@6.1.0: {} + node-addon-api@8.5.0: {} node-api-headers@1.8.0: {} @@ -10525,6 +11100,26 @@ snapshots: dependencies: mimic-function: 5.0.1 + onnx-proto@4.0.4: + dependencies: + protobufjs: 6.8.8 + + onnxruntime-common@1.14.0: {} + + onnxruntime-node@1.14.0: + dependencies: + onnxruntime-common: 1.14.0 + optional: true + + onnxruntime-web@1.14.0: + dependencies: + flatbuffers: 1.12.0 + guid-typescript: 1.0.9 + long: 4.0.0 + onnx-proto: 4.0.4 + onnxruntime-common: 1.14.0 + platform: 1.3.6 + openai@6.10.0(ws@8.19.0)(zod@4.3.6): optionalDependencies: ws: 8.19.0 @@ -10612,6 +11207,8 @@ snapshots: - supports-color - utf-8-validate + option@0.2.4: {} + opus-decoder@0.7.11: dependencies: '@wasm-audio-decoders/common': 9.0.7 @@ -10756,8 +11353,7 @@ snapshots: partial-json@0.1.7: {} - path-is-absolute@1.0.1: - optional: true + path-is-absolute@1.0.1: {} path-key@3.1.1: {} @@ -10777,6 +11373,15 @@ snapshots: pathe@2.0.3: {} + pdf-parse@2.4.5: + dependencies: + '@napi-rs/canvas': 0.1.80 + pdfjs-dist: 5.4.296 + + pdfjs-dist@5.4.296: + optionalDependencies: + '@napi-rs/canvas': 0.1.95 + pdfjs-dist@5.4.624: optionalDependencies: '@napi-rs/canvas': 0.1.95 @@ -10818,6 +11423,8 @@ snapshots: dependencies: pngjs: 7.0.0 + platform@1.3.6: {} + playwright-core@1.58.2: {} playwright@1.58.2: @@ -10836,6 +11443,21 @@ snapshots: postgres@3.4.8: {} + prebuild-install@7.1.3: + dependencies: + detect-libc: 2.1.2 + expand-template: 2.0.3 + github-from-package: 0.0.0 + minimist: 1.2.8 + mkdirp-classic: 0.5.3 + napi-build-utils: 2.0.0 + node-abi: 3.87.0 + pump: 3.0.3 + rc: 1.2.8 + simple-get: 4.0.1 + tar-fs: 2.1.4 + tunnel-agent: 0.6.0 + pretty-bytes@6.1.1: {} pretty-ms@8.0.0: @@ -10855,6 +11477,11 @@ snapshots: process-warning@5.0.0: {} + prom-client@15.1.3: + dependencies: + '@opentelemetry/api': 1.9.0 + tdigest: 0.1.2 + proper-lockfile@4.1.2: dependencies: graceful-fs: 4.2.11 @@ -10999,7 +11626,6 @@ snapshots: inherits: 2.0.4 string_decoder: 1.3.0 util-deprecate: 1.0.2 - optional: true readdirp@5.0.0: {} @@ -11123,6 +11749,10 @@ snapshots: transitivePeerDependencies: - supports-color + rxjs@7.8.2: + dependencies: + tslib: 2.8.1 + safe-buffer@5.1.2: {} safe-buffer@5.2.1: {} @@ -11208,6 +11838,21 @@ snapshots: setprototypeof@1.2.0: {} + sharp@0.32.6: + dependencies: + color: 4.2.3 + detect-libc: 2.1.2 + node-addon-api: 6.1.0 + prebuild-install: 7.1.3 + semver: 7.7.4 + simple-get: 4.0.1 + tar-fs: 3.1.1 + tunnel-agent: 0.6.0 + transitivePeerDependencies: + - bare-abort-controller + - bare-buffer + - react-native-b4a + sharp@0.34.5: dependencies: '@img/colour': 1.0.0 @@ -11285,6 +11930,14 @@ snapshots: dependencies: signal-polyfill: 0.2.2 + simple-concat@1.0.1: {} + + simple-get@4.0.1: + dependencies: + decompress-response: 6.0.0 + once: 1.4.0 + simple-concat: 1.0.1 + simple-git@3.32.3: dependencies: '@kwsites/file-exists': 1.1.1 @@ -11293,6 +11946,10 @@ snapshots: transitivePeerDependencies: - supports-color + simple-swizzle@0.2.4: + dependencies: + is-arrayish: 0.3.4 + simple-yenc@1.0.4: optional: true @@ -11354,6 +12011,8 @@ snapshots: split2@4.2.0: {} + sprintf-js@1.0.3: {} + sqlite-vec-darwin-arm64@0.1.7-alpha.2: optional: true @@ -11451,7 +12110,6 @@ snapshots: string_decoder@1.3.0: dependencies: safe-buffer: 5.2.1 - optional: true strip-ansi@6.0.1: dependencies: @@ -11478,6 +12136,33 @@ snapshots: array-back: 6.2.2 wordwrapjs: 5.1.1 + tar-fs@2.1.4: + dependencies: + chownr: 1.1.4 + mkdirp-classic: 0.5.3 + pump: 3.0.3 + tar-stream: 2.2.0 + + tar-fs@3.1.1: + dependencies: + pump: 3.0.3 + tar-stream: 3.1.7 + optionalDependencies: + bare-fs: 4.5.5 + bare-path: 3.0.0 + transitivePeerDependencies: + - bare-abort-controller + - bare-buffer + - react-native-b4a + + tar-stream@2.2.0: + dependencies: + bl: 4.1.0 + end-of-stream: 1.4.5 + fs-constants: 1.0.0 + inherits: 2.0.4 + readable-stream: 3.6.2 + tar-stream@3.1.7: dependencies: b4a: 1.8.0 @@ -11495,6 +12180,18 @@ snapshots: minizlib: 3.1.0 yallist: 5.0.0 + tdigest@0.1.2: + dependencies: + bintrees: 1.0.2 + + teex@1.0.1: + dependencies: + streamx: 2.23.0 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + optional: true + text-decoder@1.2.7: dependencies: b4a: 1.8.0 @@ -11625,12 +12322,16 @@ snapshots: '@quansync/fs': 1.0.0 quansync: 1.0.0 + underscore@1.13.8: {} + undici-types@6.21.0: {} undici-types@7.16.0: {} undici-types@7.18.2: {} + undici@6.23.0: {} + undici@7.22.0: {} universal-github-app-jwt@2.2.2: {} @@ -11774,6 +12475,8 @@ snapshots: ws@8.19.0: {} + xmlbuilder@10.1.1: {} + y18n@5.0.8: {} yallist@4.0.0: {} diff --git a/src/agents/agent-scope.ts b/src/agents/agent-scope.ts index bdc8806569696..4bddad251cf24 100644 --- a/src/agents/agent-scope.ts +++ b/src/agents/agent-scope.ts @@ -31,6 +31,8 @@ type ResolvedAgentConfig = { model?: AgentEntry["model"]; skills?: AgentEntry["skills"]; memorySearch?: AgentEntry["memorySearch"]; + deepMemory?: AgentEntry["deepMemory"]; + rustfs?: AgentEntry["rustfs"]; humanDelay?: AgentEntry["humanDelay"]; heartbeat?: AgentEntry["heartbeat"]; identity?: AgentEntry["identity"]; @@ -133,6 +135,8 @@ export function resolveAgentConfig( : undefined, skills: Array.isArray(entry.skills) ? entry.skills : undefined, memorySearch: entry.memorySearch, + deepMemory: entry.deepMemory, + rustfs: entry.rustfs, humanDelay: entry.humanDelay, heartbeat: entry.heartbeat, identity: entry.identity, diff --git a/src/agents/deep-memory.ts b/src/agents/deep-memory.ts new file mode 100644 index 0000000000000..e1d85a98dab65 --- /dev/null +++ b/src/agents/deep-memory.ts @@ -0,0 +1,138 @@ +import type { OpenClawConfig } from "../config/config.js"; +import type { DeepMemoryConfig } from "../config/types.tools.js"; +import { clampInt } from "../utils.js"; +import { resolveAgentConfig } from "./agent-scope.js"; + +export type ResolvedDeepMemoryConfig = { + enabled: boolean; + namespace: string; + apiKey?: string; + baseUrl: string; + timeoutMs: number; + retrieve: { + maxMemories: number; + cache: { enabled: boolean; ttlMs: number; maxEntries: number }; + }; + inject: { maxChars: number; label: string }; + update: { + enabled: boolean; + thresholds: { deltaBytes: number; deltaMessages: number }; + debounceMs: number; + nearCompaction: { enabled: boolean }; + }; +}; + +const DEFAULT_RETRIEVE_MAX_MEMORIES = 10; +const DEFAULT_RETRIEVE_CACHE_TTL_MINUTES = 5; +const DEFAULT_RETRIEVE_CACHE_MAX_ENTRIES = 200; +const DEFAULT_INJECT_MAX_CHARS = 4000; +const DEFAULT_TIMEOUT_SECONDS = 2; +const DEFAULT_UPDATE_DEBOUNCE_MS = 5000; +const DEFAULT_UPDATE_DELTA_BYTES = 100_000; +const DEFAULT_UPDATE_DELTA_MESSAGES = 50; +const DEFAULT_NAMESPACE = "default"; + +function mergeConfig( + defaults: DeepMemoryConfig | undefined, + overrides: DeepMemoryConfig | undefined, +): DeepMemoryConfig { + // Shallow merge is sufficient here; we normalize downstream. + return { + ...defaults, + ...overrides, + retrieve: { + ...defaults?.retrieve, + ...overrides?.retrieve, + cache: { + ...defaults?.retrieve?.cache, + ...overrides?.retrieve?.cache, + }, + }, + inject: { + ...defaults?.inject, + ...overrides?.inject, + }, + update: { + ...defaults?.update, + ...overrides?.update, + thresholds: { + ...defaults?.update?.thresholds, + ...overrides?.update?.thresholds, + }, + nearCompaction: { + ...defaults?.update?.nearCompaction, + ...overrides?.update?.nearCompaction, + }, + }, + }; +} + +export function resolveDeepMemoryConfig( + cfg: OpenClawConfig, + agentId: string, +): ResolvedDeepMemoryConfig | null { + const defaults = cfg.agents?.defaults?.deepMemory; + const overrides = resolveAgentConfig(cfg, agentId)?.deepMemory; + const merged = mergeConfig(defaults, overrides); + + const enabled = merged.enabled ?? false; + const namespace = (merged.namespace ?? DEFAULT_NAMESPACE).trim() || DEFAULT_NAMESPACE; + const apiKey = (merged.apiKey ?? "").trim() || undefined; + const baseUrl = (merged.baseUrl ?? "").trim(); + if (!enabled || !baseUrl) { + return null; + } + + const timeoutSeconds = merged.timeoutSeconds ?? DEFAULT_TIMEOUT_SECONDS; + const timeoutMs = clampInt(Math.floor(timeoutSeconds * 1000), 100, 60_000); + + const maxMemories = clampInt( + merged.retrieve?.maxMemories ?? DEFAULT_RETRIEVE_MAX_MEMORIES, + 1, + 50, + ); + const cacheEnabled = merged.retrieve?.cache?.enabled ?? true; + const ttlMinutes = merged.retrieve?.cache?.ttlMinutes ?? DEFAULT_RETRIEVE_CACHE_TTL_MINUTES; + const ttlMs = clampInt(Math.floor(ttlMinutes * 60_000), 0, 60 * 60_000); + const maxEntries = clampInt( + merged.retrieve?.cache?.maxEntries ?? DEFAULT_RETRIEVE_CACHE_MAX_ENTRIES, + 1, + 5000, + ); + + const injectMaxChars = clampInt(merged.inject?.maxChars ?? DEFAULT_INJECT_MAX_CHARS, 200, 50_000); + const label = (merged.inject?.label ?? "Deep Memory").trim() || "Deep Memory"; + + const updateEnabled = merged.update?.enabled ?? true; + const deltaBytes = clampInt( + merged.update?.thresholds?.deltaBytes ?? DEFAULT_UPDATE_DELTA_BYTES, + 0, + Number.MAX_SAFE_INTEGER, + ); + const deltaMessages = clampInt( + merged.update?.thresholds?.deltaMessages ?? DEFAULT_UPDATE_DELTA_MESSAGES, + 0, + Number.MAX_SAFE_INTEGER, + ); + const debounceMs = clampInt(merged.update?.debounceMs ?? DEFAULT_UPDATE_DEBOUNCE_MS, 0, 60_000); + const nearCompactionEnabled = merged.update?.nearCompaction?.enabled ?? true; + + return { + enabled: true, + namespace, + apiKey, + baseUrl, + timeoutMs, + retrieve: { + maxMemories, + cache: { enabled: Boolean(cacheEnabled), ttlMs, maxEntries }, + }, + inject: { maxChars: injectMaxChars, label }, + update: { + enabled: Boolean(updateEnabled), + thresholds: { deltaBytes, deltaMessages }, + debounceMs, + nearCompaction: { enabled: Boolean(nearCompactionEnabled) }, + }, + }; +} diff --git a/src/agents/openclaw-tools.ts b/src/agents/openclaw-tools.ts index 22140d167a917..705f8253c0467 100644 --- a/src/agents/openclaw-tools.ts +++ b/src/agents/openclaw-tools.ts @@ -9,6 +9,11 @@ import { createBrowserTool } from "./tools/browser-tool.js"; import { createCanvasTool } from "./tools/canvas-tool.js"; import type { AnyAgentTool } from "./tools/common.js"; import { createCronTool } from "./tools/cron-tool.js"; +import { + createFileIngestTool, + createFileSearchTool, + createFileSendTool, +} from "./tools/file-tools.js"; import { createGatewayTool } from "./tools/gateway-tool.js"; import { createImageTool } from "./tools/image-tool.js"; import { createMessageTool } from "./tools/message-tool.js"; @@ -108,6 +113,19 @@ export function createOpenClawTools(options?: { requireExplicitTarget: options?.requireExplicitMessageTarget, requesterSenderId: options?.requesterSenderId ?? undefined, }); + const fileSearchTool = createFileSearchTool({ + config: options?.config, + agentSessionKey: options?.agentSessionKey, + }); + const fileSendTool = createFileSendTool({ + config: options?.config, + agentSessionKey: options?.agentSessionKey, + }); + const fileIngestTool = createFileIngestTool({ + config: options?.config, + agentSessionKey: options?.agentSessionKey, + workspaceDir: options?.workspaceDir, + }); const tools: AnyAgentTool[] = [ createBrowserTool({ sandboxBridgeUrl: options?.sandboxBrowserBridgeUrl, @@ -170,6 +188,9 @@ export function createOpenClawTools(options?: { agentSessionKey: options?.agentSessionKey, config: options?.config, }), + ...(fileSearchTool ? [fileSearchTool] : []), + ...(fileSendTool ? [fileSendTool] : []), + ...(fileIngestTool ? [fileIngestTool] : []), ...(webSearchTool ? [webSearchTool] : []), ...(webFetchTool ? [webFetchTool] : []), ...(imageTool ? [imageTool] : []), diff --git a/src/agents/rustfs.ts b/src/agents/rustfs.ts new file mode 100644 index 0000000000000..e17788e87fa01 --- /dev/null +++ b/src/agents/rustfs.ts @@ -0,0 +1,57 @@ +import type { OpenClawConfig } from "../config/config.js"; +import type { RustFsConfig } from "../config/types.tools.js"; +import { clampInt } from "../utils.js"; +import { resolveAgentConfig } from "./agent-scope.js"; + +export type ResolvedRustFsConfig = { + enabled: boolean; + project: string; + apiKey?: string; + baseUrl: string; + linkTtlSeconds: number; + maxUploadBytes: number; +}; + +const DEFAULT_PROJECT = "default"; +const DEFAULT_LINK_TTL_SECONDS = 300; +const DEFAULT_MAX_UPLOAD_BYTES = 500 * 1024 * 1024; + +function mergeConfig( + defaults: RustFsConfig | undefined, + overrides: RustFsConfig | undefined, +): RustFsConfig { + return { ...defaults, ...overrides }; +} + +export function resolveRustFsConfig( + cfg: OpenClawConfig, + agentId: string, +): ResolvedRustFsConfig | null { + const defaults = cfg.agents?.defaults?.rustfs; + const overrides = resolveAgentConfig(cfg, agentId)?.rustfs; + const merged = mergeConfig(defaults, overrides); + + const enabled = merged.enabled ?? false; + const baseUrl = (merged.baseUrl ?? "").trim(); + if (!enabled || !baseUrl) { + return null; + } + + const project = (merged.project ?? DEFAULT_PROJECT).trim() || DEFAULT_PROJECT; + const apiKey = (merged.apiKey ?? "").trim() || undefined; + const linkTtlSeconds = clampInt(merged.linkTtlSeconds ?? DEFAULT_LINK_TTL_SECONDS, 30, 3600); + const maxUploadBytes = clampInt( + merged.maxUploadBytes ?? DEFAULT_MAX_UPLOAD_BYTES, + 1, + 10 * 1024 * 1024 * 1024, + ); + + return { + enabled: true, + project, + apiKey, + baseUrl, + linkTtlSeconds, + maxUploadBytes, + }; +} diff --git a/src/agents/tools/file-tools.test.ts b/src/agents/tools/file-tools.test.ts new file mode 100644 index 0000000000000..9a342c5a800a0 --- /dev/null +++ b/src/agents/tools/file-tools.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, test } from "vitest"; +import type { OpenClawConfig } from "../../config/config.js"; +import { resolveRustFsConfig } from "../rustfs.js"; +import { createFileIngestTool } from "./file-tools.js"; + +describe("resolveRustFsConfig", () => { + test("returns null when disabled or baseUrl missing", () => { + const cfg = { + agents: { defaults: { rustfs: { enabled: false, baseUrl: "http://localhost:8099" } } }, + } as unknown as OpenClawConfig; + expect(resolveRustFsConfig(cfg, "agent")).toBeNull(); + + const cfg2 = { + agents: { defaults: { rustfs: { enabled: true, baseUrl: "" } } }, + } as unknown as OpenClawConfig; + expect(resolveRustFsConfig(cfg2, "agent")).toBeNull(); + }); + + test("merges defaults + overrides and clamps values", () => { + const cfg = { + agents: { + defaults: { + rustfs: { + enabled: true, + baseUrl: "http://localhost:8099", + project: "default", + linkTtlSeconds: 1, + maxUploadBytes: 999_999_999_999, + }, + }, + list: [ + { + id: "a", + default: true, + rustfs: { project: "p2", linkTtlSeconds: 99999, maxUploadBytes: 0 }, + }, + ], + }, + } as unknown as OpenClawConfig; + + const out = resolveRustFsConfig(cfg, "a"); + expect(out).not.toBeNull(); + expect(out?.project).toBe("p2"); + expect(out?.linkTtlSeconds).toBe(3600); + expect(out?.maxUploadBytes).toBe(1); + }); +}); + +describe("file_ingest tool schema", () => { + test("includes semantic hint fields", () => { + const cfg = { + agents: { + defaults: { + rustfs: { + enabled: true, + baseUrl: "http://localhost:8099", + project: "default", + }, + }, + list: [{ id: "a", default: true }], + }, + } as unknown as OpenClawConfig; + + const tool = createFileIngestTool({ + config: cfg, + agentSessionKey: "session", + workspaceDir: "/tmp/workspace", + }); + expect(tool?.name).toBe("file_ingest"); + + const props = (tool as unknown as { parameters?: { properties?: Record } }) + .parameters?.properties; + expect(props).toBeTruthy(); + expect(props).toHaveProperty("kind"); + expect(props).toHaveProperty("tags"); + expect(props).toHaveProperty("hint"); + }); +}); + +describe("file_search tool schema", () => { + test("includes semantic retrieval options", async () => { + const mod = await import("./file-tools.js"); + const cfg = { + agents: { + defaults: { + rustfs: { + enabled: true, + baseUrl: "http://localhost:8099", + project: "default", + }, + }, + list: [{ id: "a", default: true }], + }, + } as unknown as OpenClawConfig; + + const tool = mod.createFileSearchTool({ config: cfg, agentSessionKey: "session" }); + expect(tool?.name).toBe("file_search"); + const props = (tool as unknown as { parameters?: { properties?: Record } }) + .parameters?.properties; + expect(props).toBeTruthy(); + expect(props).toHaveProperty("includeSemantic"); + expect(props).toHaveProperty("semanticQuery"); + expect(props).toHaveProperty("semanticMaxFiles"); + expect(props).toHaveProperty("semanticMaxMemories"); + expect(props).toHaveProperty("semanticMaxChars"); + expect(props).toHaveProperty("rerank"); + expect(props).toHaveProperty("extractStatus"); + }); +}); + +describe("file_search clarify behavior", () => { + test("close semantic scores triggers clarify with topic hint when present", async () => { + const mod = await import("./file-tools.js"); + const clarify = (mod as unknown as { __test__?: { buildClarify?: unknown } }).__test__ + ?.buildClarify as + | ((params: { + candidates: Array<{ + semanticScore?: number; + semanticTopics?: string[]; + filename: string; + }>; + includeSemantic: boolean; + }) => { required: boolean; reasons: string[]; questions: string[] } | undefined) + | undefined; + expect(typeof clarify).toBe("function"); + const out = clarify?.({ + includeSemantic: true, + candidates: [ + { filename: "a.md", semanticScore: 10, semanticTopics: ["标书", "报价"] }, + { filename: "b.md", semanticScore: 9.8, semanticTopics: ["报告", "复盘"] }, + ], + }); + expect(out?.required).toBe(true); + expect(out?.reasons).toContain("close_semantic_scores"); + expect(out?.questions.join("\n")).toContain("候选1主题"); + }); +}); diff --git a/src/agents/tools/file-tools.ts b/src/agents/tools/file-tools.ts new file mode 100644 index 0000000000000..d5e9b134e1ce0 --- /dev/null +++ b/src/agents/tools/file-tools.ts @@ -0,0 +1,705 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { Type } from "@sinclair/typebox"; +import type { OpenClawConfig } from "../../config/config.js"; +import { DeepMemoryClient } from "../../deep-memory/client.js"; +import { RustFsClient } from "../../rustfs/client.js"; +import { resolveSessionAgentId } from "../agent-scope.js"; +import { resolveDeepMemoryConfig } from "../deep-memory.js"; +import { resolveRustFsConfig } from "../rustfs.js"; +import type { AnyAgentTool } from "./common.js"; +import { jsonResult, readNumberParam, readStringParam } from "./common.js"; + +type FileSearchCandidate = { + n: number; + fileId: string; + filename: string; + mime?: string; + size?: number; + createdAtMs?: number; + extractStatus?: string; + tags?: string[]; + kind?: string; + hint?: string; + semanticScore?: number; + semanticTopics?: string[]; + semanticEntities?: string[]; + semanticContext?: string; + semanticSummary?: string; +}; + +function buildClarify(params: { candidates: FileSearchCandidate[]; includeSemantic: boolean }): + | { + required: boolean; + reasons: string[]; + questions: string[]; + } + | undefined { + const c = params.candidates; + if (c.length === 0) { + return { + required: false, + reasons: ["no_candidates"], + questions: ["没找到匹配文件。你能补充文件类型、时间范围、或更具体的关键词吗?"], + }; + } + + const reasons: string[] = []; + const questions: string[] = []; + + // Same filename ambiguity. + const byName = new Map(); + for (const it of c) { + const key = (it.filename ?? "").trim().toLowerCase(); + if (!key) { + continue; + } + const arr = byName.get(key) ?? []; + arr.push(it); + byName.set(key, arr); + } + for (const [name, arr] of byName.entries()) { + if (arr.length >= 2) { + reasons.push("duplicate_filename"); + questions.push( + `发现多个同名文件 \`${name}\`。你要哪一个(按 n 选择),或提供更具体的会话/时间线索?`, + ); + break; + } + } + + if (params.includeSemantic && c.length >= 2) { + const top = c[0]; + const second = c[1]; + const s1 = typeof top?.semanticScore === "number" ? top.semanticScore : 0; + const s2 = typeof second?.semanticScore === "number" ? second.semanticScore : 0; + if (s1 <= 0) { + reasons.push("weak_semantic_evidence"); + questions.push("语义证据较弱。你能提供更明确的项目/主题/关键词,或文件的大致名称吗?"); + } else if (s2 > 0 && s1 - s2 <= Math.max(0.15 * s1, 0.5)) { + reasons.push("close_semantic_scores"); + const tTopics = (top?.semanticTopics ?? []).slice(0, 3).join("、"); + const sTopics = (second?.semanticTopics ?? []).slice(0, 3).join("、"); + if (tTopics || sTopics) { + questions.push( + `前两个候选相关性接近。候选1主题:${tTopics || "(无)"};候选2主题:${sTopics || "(无)"}。你更倾向哪一个(按 n 选择)?`, + ); + } else { + questions.push( + "前两个候选相关性接近。你更倾向哪一个(按 n 选择),或说出你要的具体内容点?", + ); + } + } + } + + // If multiple candidates and no disambiguation info, encourage choosing by n. + if (c.length >= 2) { + reasons.push("multiple_candidates"); + questions.push("请从 candidates 里回复 n(或 fileId)确认要发送的文件。"); + } + + if (reasons.length === 0) { + return undefined; + } + return { + required: true, + reasons: Array.from(new Set(reasons)), + questions: Array.from(new Set(questions)).slice(0, 5), + }; +} + +function readSemanticsFromAnnotations(annotations: unknown): { + summary?: string; + topics?: string[]; + entities?: string[]; +} { + const root = + annotations && typeof annotations === "object" + ? (annotations as Record) + : null; + const sem = + root && root.semantics && typeof root.semantics === "object" + ? (root.semantics as Record) + : null; + if (!sem) { + return {}; + } + const summary = typeof sem.summary === "string" ? sem.summary.slice(0, 400) : undefined; + const topics = + Array.isArray(sem.topics) && sem.topics.every((t) => typeof t === "string") + ? sem.topics.slice(0, 10) + : undefined; + const entities = + Array.isArray(sem.entities) && sem.entities.every((t) => typeof t === "string") + ? sem.entities.slice(0, 10) + : undefined; + return { + summary: summary && summary.trim() ? summary.trim() : undefined, + topics: topics && topics.length > 0 ? topics : undefined, + entities: entities && entities.length > 0 ? entities : undefined, + }; +} + +function readIngestHintsFromAnnotations(annotations: unknown): { + tags?: string[]; + kind?: string; + hint?: string; +} { + const root = + annotations && typeof annotations === "object" + ? (annotations as Record) + : null; + const ingestRaw = + root && root.openclaw_ingest && typeof root.openclaw_ingest === "object" + ? (root.openclaw_ingest as Record) + : null; + const clsRaw = + root && root.classification && typeof root.classification === "object" + ? (root.classification as Record) + : null; + const ingest = ingestRaw ?? clsRaw; + if (!ingest) { + return {}; + } + const kind = typeof ingest.kind === "string" ? ingest.kind.trim() : ""; + const hint = typeof ingest.hint === "string" ? ingest.hint.trim() : ""; + const tags = + Array.isArray(ingest.tags) && ingest.tags.every((t) => typeof t === "string") + ? ingest.tags + .map((t) => t.trim()) + .filter(Boolean) + .slice(0, 20) + : undefined; + return { + kind: kind || undefined, + hint: hint || undefined, + tags: tags && tags.length > 0 ? tags : undefined, + }; +} + +function buildCandidates(items: Array>): FileSearchCandidate[] { + return items.slice(0, 50).map((raw, idx) => { + const fileId = typeof raw.file_id === "string" ? raw.file_id : ""; + const filename = typeof raw.filename === "string" ? raw.filename : ""; + const mime = typeof raw.mime === "string" ? raw.mime : undefined; + const size = typeof raw.size === "number" ? raw.size : undefined; + const createdAtMs = typeof raw.created_at_ms === "number" ? raw.created_at_ms : undefined; + const extractStatus = typeof raw.extract_status === "string" ? raw.extract_status : undefined; + + const { tags, kind, hint } = readIngestHintsFromAnnotations(raw.annotations); + const sem = readSemanticsFromAnnotations(raw.annotations); + + const semantic = + raw.semantic && typeof raw.semantic === "object" + ? (raw.semantic as Record) + : null; + const semanticScore = + semantic && typeof semantic.score === "number" ? semantic.score : undefined; + const semanticTopics = + (semantic && Array.isArray(semantic.topics) + ? (semantic.topics as string[]).filter((t) => typeof t === "string").slice(0, 10) + : undefined) ?? sem.topics; + const semanticEntities = + (semantic && Array.isArray(semantic.entities) + ? (semantic.entities as string[]).filter((t) => typeof t === "string").slice(0, 10) + : undefined) ?? sem.entities; + const semanticContext = + semantic && typeof semantic.context === "string" ? semantic.context.slice(0, 400) : undefined; + const semanticSummary = + (semantic && typeof semantic.summary === "string" + ? semantic.summary.slice(0, 400) + : undefined) ?? sem.summary; + + return { + n: idx + 1, + fileId, + filename, + mime, + size, + createdAtMs, + extractStatus, + tags, + kind, + hint, + semanticScore, + semanticTopics, + semanticEntities, + semanticContext, + semanticSummary, + }; + }); +} + +const FileSearchSchema = Type.Object({ + query: Type.Optional(Type.String()), + semanticQuery: Type.Optional(Type.String()), + sessionId: Type.Optional(Type.String()), + mime: Type.Optional(Type.String()), + extractStatus: Type.Optional(Type.String()), + limit: Type.Optional(Type.Number()), + includeSemantic: Type.Optional(Type.Boolean()), + semanticMaxFiles: Type.Optional(Type.Number()), + semanticMaxMemories: Type.Optional(Type.Number()), + semanticMaxChars: Type.Optional(Type.Number()), + rerank: Type.Optional(Type.Boolean()), +}); + +const FileSendSchema = Type.Object({ + fileId: Type.String(), + ttlSeconds: Type.Optional(Type.Number()), +}); + +const FileIngestSchema = Type.Object({ + path: Type.String(), + sessionId: Type.Optional(Type.String()), + source: Type.Optional(Type.String()), + mime: Type.Optional(Type.String()), + kind: Type.Optional(Type.String()), + tags: Type.Optional(Type.Array(Type.String())), + hint: Type.Optional(Type.String()), +}); + +function buildClient(cfg: OpenClawConfig, agentId: string): RustFsClient | null { + const resolved = resolveRustFsConfig(cfg, agentId); + if (!resolved) { + return null; + } + return new RustFsClient({ + baseUrl: resolved.baseUrl, + apiKey: resolved.apiKey, + project: resolved.project, + linkTtlSeconds: resolved.linkTtlSeconds, + maxUploadBytes: resolved.maxUploadBytes, + }); +} + +function buildDeepMemoryClient(cfg: OpenClawConfig, agentId: string): DeepMemoryClient | null { + const resolved = resolveDeepMemoryConfig(cfg, agentId); + if (!resolved) { + return null; + } + return new DeepMemoryClient({ + baseUrl: resolved.baseUrl, + timeoutMs: resolved.timeoutMs, + cache: resolved.retrieve.cache, + namespace: resolved.namespace, + apiKey: resolved.apiKey, + }); +} + +export function createFileSearchTool(options: { + config?: OpenClawConfig; + agentSessionKey?: string; +}): AnyAgentTool | null { + const cfg = options.config; + if (!cfg) { + return null; + } + const agentId = resolveSessionAgentId({ sessionKey: options.agentSessionKey, config: cfg }); + const client = buildClient(cfg, agentId); + if (!client) { + return null; + } + const deep = buildDeepMemoryClient(cfg, agentId); + const semanticCache = new Map }>(); + const semanticCacheTtlMs = 30_000; + const semanticMaxConcurrent = 4; + + async function mapWithConcurrency( + items: T[], + concurrency: number, + fn: (item: T) => Promise, + ): Promise { + const limit = Math.max(1, Math.min(16, Math.trunc(concurrency))); + const out: R[] = []; + let i = 0; + async function worker(): Promise { + for (;;) { + const idx = i; + i += 1; + const item = items[idx]; + if (!item) { + return; + } + out[idx] = await fn(item); + } + } + const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker()); + await Promise.all(workers); + return out; + } + return { + label: "File Search", + name: "file_search", + description: + "Search session files archived in RustFS by project + optional filters (query, sessionId, mime). Always ask the user to confirm a candidate (by n or fileId) before calling file_send.", + parameters: FileSearchSchema, + execute: async (_toolCallId, params) => { + try { + const query = readStringParam(params, "query", { required: false, label: "query" }); + const semanticQuery = readStringParam(params, "semanticQuery", { + required: false, + label: "semanticQuery", + }); + const sessionId = readStringParam(params, "sessionId", { + required: false, + label: "sessionId", + }); + const mime = readStringParam(params, "mime", { required: false, label: "mime" }); + const extractStatus = readStringParam(params, "extractStatus", { + required: false, + label: "extractStatus", + }); + const limit = readNumberParam(params, "limit", { integer: true }); + const includeSemantic = Boolean((params as Record).includeSemantic); + const rerank = Boolean((params as Record).rerank); + const semanticMaxFiles = + readNumberParam(params, "semanticMaxFiles", { integer: true }) ?? 10; + const semanticMaxMemories = + readNumberParam(params, "semanticMaxMemories", { integer: true }) ?? 8; + const semanticMaxChars = + readNumberParam(params, "semanticMaxChars", { integer: true }) ?? 800; + + const out = await client.search({ + query, + sessionId, + mime, + extractStatus, + limit, + }); + if (!out.ok) { + return jsonResult(out); + } + if (!includeSemantic) { + const items = out.items as unknown as Array>; + return jsonResult({ + ok: true, + items, + candidates: buildCandidates(items), + selection: { + required: true, + message: + "请让用户从 candidates 里选择(回复 n 或 fileId),确认后再调用 file_send。避免在未确认时直接发送文件。", + }, + }); + } + + if (!deep) { + const items = out.items as unknown as Array>; + return jsonResult({ + ok: true, + items, + candidates: buildCandidates(items), + semantic: { ok: false, error: "deep-memory not configured for this agent" }, + selection: { + required: true, + message: + "deep-memory 未配置;请让用户从 candidates 里选择(回复 n 或 fileId),确认后再调用 file_send。", + }, + }); + } + + const userInput = (semanticQuery ?? query ?? "").trim(); + if (!userInput) { + return jsonResult({ + ...out, + semantic: { + ok: false, + error: "semanticQuery or query required for includeSemantic=true", + }, + }); + } + + const semanticItems = out.items.slice(0, Math.max(1, Math.min(50, semanticMaxFiles))); + const semanticIds = new Set(semanticItems.map((i) => i.file_id)); + const scoreOf = (v: unknown): number => { + const score = (v as { semantic?: { score?: unknown } } | null | undefined)?.semantic + ?.score; + return typeof score === "number" ? score : 0; + }; + const now = Date.now(); + for (const [key, entry] of semanticCache.entries()) { + if (entry.expiresAt <= now) { + semanticCache.delete(key); + } + } + + const semanticLookup = await mapWithConcurrency( + semanticItems, + semanticMaxConcurrent, + async (item) => { + const dmSessionId = `rustfs:file:${item.file_id}`; + const retrieveKey = `retrieve::${userInput}::${semanticMaxMemories}::${dmSessionId}`; + const inspectKey = `inspect::${dmSessionId}`; + + const cachedRetrieve = semanticCache.get(retrieveKey); + const cachedInspect = semanticCache.get(inspectKey); + + const retrievePromise = + cachedRetrieve && cachedRetrieve.expiresAt > Date.now() + ? Promise.resolve(cachedRetrieve.value) + : deep + .retrieveContext({ + userInput, + sessionId: dmSessionId, + maxMemories: Math.max(1, Math.min(50, semanticMaxMemories)), + }) + .then((retrieved) => { + const context = (retrieved.context ?? "").trim(); + const contextClamped = + context.length > semanticMaxChars + ? context.slice(0, semanticMaxChars) + : context; + const memories = Array.isArray(retrieved.memories) ? retrieved.memories : []; + const score = memories.reduce( + (acc, m) => acc + (typeof m.relevance === "number" ? m.relevance : 0), + 0, + ); + const value = { + score, + context: contextClamped, + memories: memories + .slice(0, Math.max(1, Math.min(50, semanticMaxMemories))) + .map((m) => ({ + id: m.id, + relevance: m.relevance, + importance: m.importance, + content: + typeof m.content === "string" ? m.content.slice(0, 400) : undefined, + })), + } satisfies Record; + semanticCache.set(retrieveKey, { + value, + expiresAt: Date.now() + semanticCacheTtlMs, + }); + return value; + }); + + const inspectPromise = + cachedInspect && cachedInspect.expiresAt > Date.now() + ? Promise.resolve(cachedInspect.value) + : deep + .inspectSession({ + sessionId: dmSessionId, + limit: 100, + includeContent: false, + }) + .then((inspected) => { + const topics = + Array.isArray(inspected.topics) && inspected.topics.length > 0 + ? inspected.topics + .map((t) => (typeof t?.name === "string" ? t.name.trim() : "")) + .filter(Boolean) + .slice(0, 20) + : []; + const entities = + Array.isArray(inspected.entities) && inspected.entities.length > 0 + ? inspected.entities + .map((e) => (typeof e?.name === "string" ? e.name.trim() : "")) + .filter(Boolean) + .slice(0, 20) + : []; + const summary = + typeof inspected.summary === "string" + ? inspected.summary.slice(0, 800) + : undefined; + const value = { + topics, + entities, + summary, + } satisfies Record; + semanticCache.set(inspectKey, { + value, + expiresAt: Date.now() + semanticCacheTtlMs, + }); + return value; + }); + + const [retrieveEvidence, inspectEvidence] = await Promise.all([ + retrievePromise, + inspectPromise, + ]); + + const semantic = { + deepMemorySessionId: dmSessionId, + ...inspectEvidence, + ...retrieveEvidence, + } satisfies Record; + + return { fileId: item.file_id, semantic }; + }, + ); + + const semanticMap = new Map>(); + for (const entry of semanticLookup) { + if (entry && entry.fileId && entry.semantic) { + semanticMap.set(entry.fileId, entry.semantic); + } + } + + const decorated: Array> = out.items.map((item) => { + if (!semanticIds.has(item.file_id)) { + return item as unknown as Record; + } + const semantic = semanticMap.get(item.file_id); + return semantic + ? ({ ...(item as unknown as Record), semantic } as Record< + string, + unknown + >) + : (item as unknown as Record); + }); + + const finalItems = rerank + ? decorated.toSorted((a, b) => scoreOf(b) - scoreOf(a)) + : decorated; + + const candidates = buildCandidates(finalItems); + const clarify = buildClarify({ candidates, includeSemantic }); + + return jsonResult({ + ok: true, + items: finalItems, + candidates, + semantic: { ok: true }, + clarify, + selection: { + required: true, + message: + "请让用户从 candidates 里选择(回复 n 或 fileId),确认后再调用 file_send。若用户目标不明确,先追问澄清再发送。", + }, + }); + } catch (err) { + return jsonResult({ ok: false, error: err instanceof Error ? err.message : String(err) }); + } + }, + }; +} + +export function createFileSendTool(options: { + config?: OpenClawConfig; + agentSessionKey?: string; +}): AnyAgentTool | null { + const cfg = options.config; + if (!cfg) { + return null; + } + const agentId = resolveSessionAgentId({ sessionKey: options.agentSessionKey, config: cfg }); + const client = buildClient(cfg, agentId); + if (!client) { + return null; + } + return { + label: "File Send (Link)", + name: "file_send", + description: + "Create a short-lived public download link for a RustFS fileId. Only call this after the user explicitly confirms which file to send (from file_search candidates).", + parameters: FileSendSchema, + execute: async (_toolCallId, params) => { + try { + const fileId = readStringParam(params, "fileId", { required: true, label: "fileId" }); + const ttlSeconds = readNumberParam(params, "ttlSeconds", { integer: true }); + const out = await client.createLink({ + fileId, + ttlSeconds: ttlSeconds ?? undefined, + }); + return jsonResult(out); + } catch (err) { + return jsonResult({ ok: false, error: err instanceof Error ? err.message : String(err) }); + } + }, + }; +} + +export function createFileIngestTool(options: { + config?: OpenClawConfig; + agentSessionKey?: string; + workspaceDir?: string; +}): AnyAgentTool | null { + const cfg = options.config; + const workspaceDir = options.workspaceDir?.trim(); + if (!cfg || !workspaceDir) { + return null; + } + const agentId = resolveSessionAgentId({ sessionKey: options.agentSessionKey, config: cfg }); + const client = buildClient(cfg, agentId); + if (!client) { + return null; + } + return { + label: "File Ingest", + name: "file_ingest", + description: + "Upload a local workspace file into RustFS (streaming) so it can be searched/shared later. Path must be within the workspace directory.", + parameters: FileIngestSchema, + execute: async (_toolCallId, params) => { + try { + const rel = readStringParam(params, "path", { required: true, label: "path" }); + const sessionId = readStringParam(params, "sessionId", { + required: false, + label: "sessionId", + }); + const source = readStringParam(params, "source", { required: false, label: "source" }); + const mime = readStringParam(params, "mime", { required: false, label: "mime" }); + const kind = readStringParam(params, "kind", { required: false, label: "kind" }); + const hint = readStringParam(params, "hint", { required: false, label: "hint" }); + const tagsRaw = (params as Record).tags; + const tags = Array.isArray(tagsRaw) + ? tagsRaw + .filter((t): t is string => typeof t === "string") + .map((t) => t.trim()) + .filter((t) => t.length > 0) + .slice(0, 50) + : undefined; + + const absPath = path.isAbsolute(rel) ? path.resolve(rel) : path.resolve(workspaceDir, rel); + const relToWorkspace = path.relative(workspaceDir, absPath).replace(/\\/g, "/"); + const inWorkspace = + relToWorkspace.length > 0 && + !relToWorkspace.startsWith("..") && + !path.isAbsolute(relToWorkspace); + if (!inWorkspace) { + return jsonResult({ ok: false, error: "path must be within the workspace" }); + } + + const stat = await fs.stat(absPath).catch(() => null); + if (!stat || !stat.isFile()) { + return jsonResult({ ok: false, error: "file not found" }); + } + + const out = await client.ingestFile({ + absPath, + sessionId: sessionId ?? undefined, + source: source ?? undefined, + mime: mime ?? undefined, + }); + if (out.ok && (kind || hint || (tags && tags.length > 0))) { + await client.upsertAnnotations({ + fileId: out.file_id, + source: "openclaw", + annotations: { + openclaw_ingest: { + kind: kind ?? undefined, + hint: hint ?? undefined, + tags: tags && tags.length > 0 ? tags : undefined, + session_id: sessionId ?? undefined, + source: source ?? undefined, + mime: mime ?? undefined, + }, + }, + }); + } + return jsonResult(out); + } catch (err) { + return jsonResult({ ok: false, error: err instanceof Error ? err.message : String(err) }); + } + }, + }; +} + +export const __test__ = { + buildClarify, + buildCandidates, + readIngestHintsFromAnnotations, + readSemanticsFromAnnotations, +}; diff --git a/src/auto-reply/commands-registry.data.ts b/src/auto-reply/commands-registry.data.ts index bdefb3ba16c25..bd26bd1ab0e91 100644 --- a/src/auto-reply/commands-registry.data.ts +++ b/src/auto-reply/commands-registry.data.ts @@ -188,6 +188,22 @@ function buildChatCommands(): ChatCommandDefinition[] { acceptsArgs: true, category: "management", }), + defineChatCommand({ + key: "forget", + nativeName: "forget", + description: "Delete deep memory by session or memory id (admin).", + textAlias: "/forget", + acceptsArgs: true, + category: "management", + }), + defineChatCommand({ + key: "deepmemory", + nativeName: "deepmemory", + description: "Show deep-memory-server health/readiness/queue status (admin).", + textAliases: ["/deepmemory", "/deep-memory"], + acceptsArgs: true, + category: "status", + }), defineChatCommand({ key: "context", nativeName: "context", diff --git a/src/auto-reply/reply/commands-core.ts b/src/auto-reply/reply/commands-core.ts index 8f64defc5eb9c..19ce9816f7fd1 100644 --- a/src/auto-reply/reply/commands-core.ts +++ b/src/auto-reply/reply/commands-core.ts @@ -10,6 +10,8 @@ import { handleApproveCommand } from "./commands-approve.js"; import { handleBashCommand } from "./commands-bash.js"; import { handleCompactCommand } from "./commands-compact.js"; import { handleConfigCommand, handleDebugCommand } from "./commands-config.js"; +import { handleDeepMemoryStatusCommand } from "./commands-deepmemory.js"; +import { handleForgetCommand } from "./commands-forget.js"; import { handleCommandsListCommand, handleContextCommand, @@ -147,6 +149,8 @@ export async function handleCommands(params: HandleCommandsParams): Promise t.trim()); + const first = tokens[0]?.toLowerCase(); + if (first && first !== "status") { + return { ok: false, error: "Usage: /deepmemory status [details] [queue]" }; + } + const details = tokens.some((t) => { + const v = t.toLowerCase(); + return v === "details" || v === "--details"; + }); + const queue = tokens.some((t) => { + const v = t.toLowerCase(); + return v === "queue" || v === "--queue"; + }); + return { ok: true, details, queue }; +} + +function formatElevatedRequired(params: Parameters[0]): string { + const lines: string[] = []; + lines.push("⚠️ /deepmemory status 需要 elevated 权限(并且 sender 在 allowFrom 中)。"); + lines.push("用法:"); + lines.push("- /deepmemory status"); + lines.push("- /deepmemory status details (尝试调用 /health/details;需要 admin key)"); + lines.push("- /deepmemory status queue (尝试调用 /queue/stats;需要 admin key)"); + if (params.elevated.failures.length > 0) { + lines.push( + `Failing gates: ${params.elevated.failures.map((f) => `${f.gate} (${f.key})`).join(", ")}`, + ); + } + if (params.sessionKey) { + lines.push( + `See: ${formatCliCommand(`openclaw sandbox explain --session ${params.sessionKey}`)}`, + ); + } + return lines.join("\n"); +} + +function asRecord(v: unknown): Record | null { + return typeof v === "object" && v ? (v as Record) : null; +} + +function pickString(obj: Record | null, key: string): string | undefined { + const v = obj?.[key]; + return typeof v === "string" && v.trim() ? v.trim() : undefined; +} + +function pickNumber(obj: Record | null, key: string): number | undefined { + const v = obj?.[key]; + return typeof v === "number" && Number.isFinite(v) ? v : undefined; +} + +export const handleDeepMemoryStatusCommand: CommandHandler = async (params, allowTextCommands) => { + if (!allowTextCommands) { + return null; + } + const parsed = parseDeepMemoryCommand(params.command.commandBodyNormalized); + if (!parsed) { + return null; + } + if (!params.command.isAuthorizedSender) { + logVerbose( + `Ignoring /deepmemory from unauthorized sender: ${params.command.senderId || ""}`, + ); + return { shouldContinue: false }; + } + if (!params.elevated.enabled || !params.elevated.allowed) { + return { shouldContinue: false, reply: { text: formatElevatedRequired(params) } }; + } + if (!parsed.ok) { + return { shouldContinue: false, reply: { text: parsed.error } }; + } + + const agentId = + params.agentId ?? + resolveSessionAgentId({ + sessionKey: params.sessionKey, + config: params.cfg, + }); + if (!agentId) { + return { + shouldContinue: false, + reply: { text: "⚠️ /deepmemory unavailable (missing agent id)." }, + }; + } + const deep = resolveDeepMemoryConfig(params.cfg, agentId); + if (!deep) { + return { + shouldContinue: false, + reply: { text: "⚠️ Deep memory 未启用(agents.*.deepMemory.enabled/baseUrl)。" }, + }; + } + + const client = new DeepMemoryClient({ + baseUrl: deep.baseUrl, + timeoutMs: deep.timeoutMs, + cache: { enabled: false, ttlMs: 0, maxEntries: 1 }, + namespace: deep.namespace, + apiKey: deep.apiKey, + }); + + const lines: string[] = []; + lines.push("Deep memory status"); + lines.push(`- baseUrl: ${deep.baseUrl}`); + lines.push(`- namespace: ${deep.namespace}`); + + const health = await client.health({ details: parsed.details }); + if (!health.ok) { + lines.push(`- health: ❌ ${health.error}`); + } else { + const body = asRecord(health.value); + const service = asRecord(body?.service); + const version = pickString(service, "version"); + const ok = body?.ok === true; + lines.push(`- health: ${ok ? "✅ ok" : "⚠️ not_ok"}${version ? ` (v${version})` : ""}`); + const guardrails = asRecord(body?.guardrails); + const rate = asRecord(guardrails?.rateLimit); + const enabled = rate?.enabled === true; + const retrieve = pickNumber(rate, "retrievePerWindow"); + const update = pickNumber(rate, "updatePerWindow"); + if (enabled) { + lines.push( + `- rateLimit: enabled retrieve=${retrieve ?? "?"}/window update=${update ?? "?"}/window`, + ); + } else { + lines.push("- rateLimit: disabled"); + } + const backlog = asRecord(guardrails?.updateBacklog); + const rejectPending = pickNumber(backlog, "rejectPending"); + if (rejectPending && rejectPending > 0) { + lines.push(`- updateBacklog: reject when pending>=${rejectPending}`); + } else { + lines.push("- updateBacklog: disabled"); + } + } + + const readyz = await client.readyz(); + if (!readyz.ok) { + lines.push(`- readyz: ❌ ${readyz.error}`); + } else { + const ok = asRecord(readyz.value)?.ok === true; + lines.push(`- readyz: ${ok ? "✅ ok" : "⚠️ not_ok"}`); + } + + if (parsed.queue) { + const q = await client.queueStats(); + if (!q.ok) { + lines.push(`- queue: ❌ ${q.error}`); + } else { + const r = asRecord(q.value); + const pending = pickNumber(r, "pendingApprox"); + const active = pickNumber(r, "active"); + lines.push( + `- queue: pendingApprox=${pending ?? "?"} active=${active ?? "?"} inflightKeys=${pickNumber(r, "inflightKeys") ?? "?"}`, + ); + } + } + + return { shouldContinue: false, reply: { text: lines.join("\n") } }; +}; diff --git a/src/auto-reply/reply/commands-forget.ts b/src/auto-reply/reply/commands-forget.ts new file mode 100644 index 0000000000000..dd85885470e55 --- /dev/null +++ b/src/auto-reply/reply/commands-forget.ts @@ -0,0 +1,224 @@ +import { resolveSessionAgentId } from "../../agents/agent-scope.js"; +import { resolveDeepMemoryConfig } from "../../agents/deep-memory.js"; +import { formatCliCommand } from "../../cli/command-format.js"; +import { DeepMemoryClient } from "../../deep-memory/client.js"; +import { logVerbose } from "../../globals.js"; +import type { CommandHandler } from "./commands-types.js"; + +type ParsedForgetCommand = + | { + ok: true; + dryRun: boolean; + confirm: boolean; + sessionId?: string; + memoryIds?: string[]; + } + | { ok: false; error: string }; + +const COMMAND = "/forget"; + +function parseForgetCommand(raw: string): ParsedForgetCommand | null { + const trimmed = raw.trim(); + if (!trimmed.toLowerCase().startsWith(COMMAND)) { + return null; + } + const rest = trimmed.slice(COMMAND.length).trim(); + if (!rest) { + return { ok: true, dryRun: true, confirm: false }; + } + const tokens = rest.split(/\s+/).filter(Boolean); + let confirm = false; + let dryRunExplicit: boolean | undefined; + let sessionId: string | undefined; + let memoryIds: string[] | undefined; + + const readRemainderList = (startIndex: number): string[] => { + const remainder = tokens.slice(startIndex).join(" ").trim(); + if (!remainder) { + return []; + } + return remainder + .split(/[,\s]+/g) + .map((s) => s.trim()) + .filter(Boolean); + }; + + for (let i = 0; i < tokens.length; i += 1) { + const t = tokens[i]?.trim(); + if (!t) { + continue; + } + const lowered = t.toLowerCase(); + if ( + lowered === "confirm" || + lowered === "--confirm" || + lowered === "yes" || + lowered === "--yes" || + lowered === "execute" || + lowered === "--execute" + ) { + confirm = true; + continue; + } + if ( + lowered === "dry-run" || + lowered === "--dry-run" || + lowered === "preview" || + lowered === "--preview" + ) { + dryRunExplicit = true; + continue; + } + if (lowered === "apply" || lowered === "--apply" || lowered === "--no-dry-run") { + dryRunExplicit = false; + continue; + } + if (lowered === "session" || lowered === "--session") { + const value = tokens[i + 1]?.trim(); + if (!value) { + return { ok: false, error: "Usage: /forget session [confirm]" }; + } + sessionId = value; + i += 1; + continue; + } + if (lowered === "id" || lowered === "--id") { + const value = tokens[i + 1]?.trim(); + if (!value) { + return { ok: false, error: "Usage: /forget id [confirm]" }; + } + memoryIds = [value]; + i += 1; + continue; + } + if (lowered === "ids" || lowered === "--ids") { + const ids = readRemainderList(i + 1); + if (ids.length === 0) { + return { ok: false, error: "Usage: /forget ids [confirm]" }; + } + memoryIds = ids; + break; + } + } + + const dryRun = dryRunExplicit ?? !confirm; + return { ok: true, dryRun, confirm, sessionId, memoryIds }; +} + +function formatElevatedRequired(params: Parameters[0]): string { + const lines: string[] = []; + lines.push("⚠️ /forget 需要 elevated 权限(并且 sender 在 allowFrom 中)。"); + lines.push("用法:"); + lines.push("- /forget (默认 dry-run,预览删除当前 session 的 deep memory)"); + lines.push("- /forget confirm (确认执行删除)"); + lines.push("- /forget id [confirm]"); + lines.push("- /forget ids [confirm]"); + lines.push("- /forget session [confirm]"); + if (params.elevated.failures.length > 0) { + lines.push( + `Failing gates: ${params.elevated.failures.map((f) => `${f.gate} (${f.key})`).join(", ")}`, + ); + } + if (params.sessionKey) { + lines.push( + `See: ${formatCliCommand(`openclaw sandbox explain --session ${params.sessionKey}`)}`, + ); + } + return lines.join("\n"); +} + +export const handleForgetCommand: CommandHandler = async (params, allowTextCommands) => { + if (!allowTextCommands) { + return null; + } + const normalized = params.command.commandBodyNormalized; + const parsed = parseForgetCommand(normalized); + if (!parsed) { + return null; + } + if (!params.command.isAuthorizedSender) { + logVerbose( + `Ignoring /forget from unauthorized sender: ${params.command.senderId || ""}`, + ); + return { shouldContinue: false }; + } + if (!params.elevated.enabled || !params.elevated.allowed) { + return { shouldContinue: false, reply: { text: formatElevatedRequired(params) } }; + } + if (!parsed.ok) { + return { shouldContinue: false, reply: { text: parsed.error } }; + } + + const agentId = + params.agentId ?? + resolveSessionAgentId({ + sessionKey: params.sessionKey, + config: params.cfg, + }); + if (!agentId) { + return { shouldContinue: false, reply: { text: "⚠️ /forget unavailable (missing agent id)." } }; + } + + const deep = resolveDeepMemoryConfig(params.cfg, agentId); + if (!deep) { + return { + shouldContinue: false, + reply: { text: "⚠️ Deep memory 未启用(agents.*.deepMemory.enabled/baseUrl)。" }, + }; + } + + const sessionIdFromContext = params.sessionEntry?.sessionId; + const sessionId = parsed.sessionId ?? sessionIdFromContext; + if (!sessionId && (!parsed.memoryIds || parsed.memoryIds.length === 0)) { + return { + shouldContinue: false, + reply: { + text: "⚠️ /forget 需要 sessionId(当前会话缺少 session id),或显式提供 memory id(s)。", + }, + }; + } + + const client = new DeepMemoryClient({ + baseUrl: deep.baseUrl, + timeoutMs: deep.timeoutMs, + cache: { enabled: false, ttlMs: 0, maxEntries: 1 }, + namespace: deep.namespace, + apiKey: deep.apiKey, + }); + + const result = await client.forget({ + sessionId: parsed.memoryIds?.length ? undefined : sessionId, + memoryIds: parsed.memoryIds, + dryRun: parsed.dryRun, + }); + if (!result.ok) { + return { + shouldContinue: false, + reply: { text: `❌ /forget 调用 deep-memory-server 失败:${result.error}` }, + }; + } + + const r = result.value; + if (r.status === "dry_run") { + const sessionLabel = sessionId ? `session=${sessionId}` : ""; + const idsLabel = r.delete_ids != null ? `ids=${r.delete_ids}` : ""; + const targets = [sessionLabel, idsLabel].filter(Boolean).join(", ") || "(none)"; + return { + shouldContinue: false, + reply: { + text: [ + `🧪 /forget dry-run: ${targets}`, + `namespace=${r.namespace ?? deep.namespace}`, + "要确认执行:/forget confirm", + ].join("\n"), + }, + }; + } + + return { + shouldContinue: false, + reply: { + text: `✅ /forget processed. namespace=${r.namespace ?? deep.namespace} deleted=${r.deleted ?? 0}`, + }, + }; +}; diff --git a/src/auto-reply/reply/get-reply-run.ts b/src/auto-reply/reply/get-reply-run.ts index 1df105427f736..e829cd2f97b57 100644 --- a/src/auto-reply/reply/get-reply-run.ts +++ b/src/auto-reply/reply/get-reply-run.ts @@ -458,6 +458,8 @@ export async function runPreparedReply( prompt: queuedBody, messageId: sessionCtx.MessageSidFull ?? sessionCtx.MessageSid, summaryLine: baseBodyTrimmedRaw, + // Clean user input for deep memory retrieval/injection (do not include system/event prefixes). + deepMemoryUserInput: baseBodyTrimmed, enqueuedAt: Date.now(), // Originating channel for reply routing. originatingChannel: ctx.OriginatingChannel, diff --git a/src/auto-reply/reply/queue/types.ts b/src/auto-reply/reply/queue/types.ts index 929f02e0726cd..519442d7d9f9b 100644 --- a/src/auto-reply/reply/queue/types.ts +++ b/src/auto-reply/reply/queue/types.ts @@ -23,6 +23,8 @@ export type FollowupRun = { /** Provider message ID, when available (for deduplication). */ messageId?: string; summaryLine?: string; + /** Clean user input used for deep-memory retrieval/injection (optional). */ + deepMemoryUserInput?: string; enqueuedAt: number; /** * Originating channel for reply routing. diff --git a/src/config/sessions/types.ts b/src/config/sessions/types.ts index c62ab8ff966c5..7d3c384cfaff0 100644 --- a/src/config/sessions/types.ts +++ b/src/config/sessions/types.ts @@ -146,6 +146,17 @@ export type SessionEntry = { compactionCount?: number; memoryFlushAt?: number; memoryFlushCompactionCount?: number; + /** + * Deep memory update bookkeeping (external long-term memory service). + * These fields are best-effort and safe to delete; they will be rehydrated. + */ + deepMemoryUpdatedAt?: number; + /** Last observed transcript byte size (for delta threshold batching). */ + deepMemoryTranscriptSize?: number; + /** Last observed transcript JSONL line count (for delta threshold batching). */ + deepMemoryTranscriptLines?: number; + /** Compaction count when we last enqueued a near-compaction deep memory update. */ + deepMemoryNearCompactionCount?: number; cliSessionIds?: Record; claudeCliSessionId?: string; label?: string; diff --git a/src/config/types.agent-defaults.ts b/src/config/types.agent-defaults.ts index 303d3b953e73c..73a6d93cef392 100644 --- a/src/config/types.agent-defaults.ts +++ b/src/config/types.agent-defaults.ts @@ -6,7 +6,7 @@ import type { HumanDelayConfig, TypingMode, } from "./types.base.js"; -import type { MemorySearchConfig } from "./types.tools.js"; +import type { DeepMemoryConfig, MemorySearchConfig, RustFsConfig } from "./types.tools.js"; export type AgentModelEntryConfig = { alias?: string; @@ -170,6 +170,10 @@ export type AgentDefaultsConfig = { }; /** Vector memory search configuration (per-agent overrides supported). */ memorySearch?: MemorySearchConfig; + /** Deep memory (external long-term memory service) configuration. */ + deepMemory?: DeepMemoryConfig; + /** RustFS (session file archive) configuration. */ + rustfs?: RustFsConfig; /** Default thinking level when no /think directive is present. */ thinkingDefault?: "off" | "minimal" | "low" | "medium" | "high" | "xhigh"; /** Default verbose level when no /verbose directive is present. */ diff --git a/src/config/types.agents.ts b/src/config/types.agents.ts index 61883abcc04db..0b642868c532e 100644 --- a/src/config/types.agents.ts +++ b/src/config/types.agents.ts @@ -15,6 +15,10 @@ export type AgentConfig = { /** Optional allowlist of skills for this agent (omit = all skills; empty = none). */ skills?: string[]; memorySearch?: MemorySearchConfig; + /** Deep memory (external long-term memory service) configuration. */ + deepMemory?: AgentDefaultsConfig["deepMemory"]; + /** RustFS (session file archive) configuration. */ + rustfs?: AgentDefaultsConfig["rustfs"]; /** Human-like delay between block replies for this agent. */ humanDelay?: HumanDelayConfig; /** Optional per-agent heartbeat overrides. */ diff --git a/src/config/types.tools.ts b/src/config/types.tools.ts index 492282f239733..56c0fcbadeffa 100644 --- a/src/config/types.tools.ts +++ b/src/config/types.tools.ts @@ -417,6 +417,95 @@ export type MemorySearchConfig = { }; }; +/** + * Deep memory configuration (optional). + * + * This is distinct from `memorySearch`: + * - memorySearch: vector/hybrid search over Markdown memory files (built-in) + * - deepMemory: external long-term memory service (graph/vector/importance), used for + * programmatic per-turn context injection + async indexing. + */ +export type DeepMemoryConfig = { + /** Enable deep memory (default: false). */ + enabled?: boolean; + /** + * Optional namespace / tenant key for isolation. + * + * Use this to prevent cross-user / cross-workspace memory mixing when multiple OpenClaw + * instances share the same deep-memory-server + stores. + */ + namespace?: string; + /** Optional shared secret for deep-memory-server (sent as x-api-key). */ + apiKey?: string; + /** Base URL for the deep memory service (e.g. http://127.0.0.1:8088). */ + baseUrl?: string; + /** Timeout for deep memory HTTP calls (seconds). */ + timeoutSeconds?: number; + /** Query-time retrieval settings (per-turn injection). */ + retrieve?: { + /** Max memories to request/format (default: 10). */ + maxMemories?: number; + /** Optional cache for retrieval results. */ + cache?: { + enabled?: boolean; + /** Cache TTL in minutes (default: 5). */ + ttlMinutes?: number; + /** Max cached entries (best-effort). */ + maxEntries?: number; + }; + }; + /** Injection formatting and bounds. */ + inject?: { + /** Max characters to inject into the system prompt (default: 4000). */ + maxChars?: number; + /** Optional label for the injected block header. */ + label?: string; + }; + /** Update/indexing settings (async, best-effort). */ + update?: { + /** Enable background updates (default: true when deepMemory.enabled). */ + enabled?: boolean; + /** Transcript delta thresholds for batching (trigger #2). */ + thresholds?: { + /** Minimum appended bytes before we enqueue an update. */ + deltaBytes?: number; + /** Minimum appended JSONL lines before we enqueue an update. */ + deltaMessages?: number; + }; + /** Debounce window (ms) to coalesce rapid triggers (default: 5000). */ + debounceMs?: number; + /** Trigger #3: enqueue an update when nearing auto-compaction (default: true). */ + nearCompaction?: { + enabled?: boolean; + }; + }; +}; + +/** + * RustFS configuration (optional). + * + * RustFS is a session file archive service (durable storage + basic metadata search). + * It is intentionally separate from deep-memory-server. + */ +export type RustFsConfig = { + /** Enable RustFS integration (default: false). */ + enabled?: boolean; + /** + * Tenant/project label for file isolation. + * + * In practice this can be a product name or project name; it maps to RustFS tenant_id. + */ + project?: string; + /** Shared secret for RustFS (sent as x-api-key). */ + apiKey?: string; + /** Base URL for RustFS (e.g. http://127.0.0.1:8099). */ + baseUrl?: string; + /** Default TTL seconds for public share links (default: 300). */ + linkTtlSeconds?: number; + /** Hard cap for uploads (bytes). Default is best-effort; RustFS also enforces its own limits. */ + maxUploadBytes?: number; +}; + export type ToolsConfig = { /** Base tool profile applied before allow/deny lists. */ profile?: ToolProfileId; diff --git a/src/config/zod-schema.agent-defaults.ts b/src/config/zod-schema.agent-defaults.ts index afbe226b0fdd9..b695f45f60a74 100644 --- a/src/config/zod-schema.agent-defaults.ts +++ b/src/config/zod-schema.agent-defaults.ts @@ -4,6 +4,8 @@ import { AgentSandboxSchema, AgentModelSchema, MemorySearchSchema, + DeepMemorySchema, + RustFsSchema, } from "./zod-schema.agent-runtime.js"; import { BlockStreamingChunkSchema, @@ -44,6 +46,8 @@ export const AgentDefaultsSchema = z contextTokens: z.number().int().positive().optional(), cliBackends: z.record(z.string(), CliBackendSchema).optional(), memorySearch: MemorySearchSchema, + deepMemory: DeepMemorySchema, + rustfs: RustFsSchema, contextPruning: z .object({ mode: z.union([z.literal("off"), z.literal("cache-ttl")]).optional(), diff --git a/src/config/zod-schema.agent-runtime.ts b/src/config/zod-schema.agent-runtime.ts index 9df0776b95602..1f58c253b796d 100644 --- a/src/config/zod-schema.agent-runtime.ts +++ b/src/config/zod-schema.agent-runtime.ts @@ -670,6 +670,71 @@ export const MemorySearchSchema = z .strict() .optional(); export { AgentModelSchema }; + +export const DeepMemorySchema = z + .object({ + enabled: z.boolean().optional(), + namespace: z.string().optional(), + apiKey: z.string().optional(), + baseUrl: z.string().optional(), + timeoutSeconds: z.number().int().positive().optional(), + retrieve: z + .object({ + maxMemories: z.number().int().positive().optional(), + cache: z + .object({ + enabled: z.boolean().optional(), + ttlMinutes: z.number().int().nonnegative().optional(), + maxEntries: z.number().int().positive().optional(), + }) + .strict() + .optional(), + }) + .strict() + .optional(), + inject: z + .object({ + maxChars: z.number().int().positive().optional(), + label: z.string().optional(), + }) + .strict() + .optional(), + update: z + .object({ + enabled: z.boolean().optional(), + thresholds: z + .object({ + deltaBytes: z.number().int().nonnegative().optional(), + deltaMessages: z.number().int().nonnegative().optional(), + }) + .strict() + .optional(), + debounceMs: z.number().int().nonnegative().optional(), + nearCompaction: z + .object({ + enabled: z.boolean().optional(), + }) + .strict() + .optional(), + }) + .strict() + .optional(), + }) + .strict() + .optional(); + +export const RustFsSchema = z + .object({ + enabled: z.boolean().optional(), + project: z.string().optional(), + apiKey: z.string().optional(), + baseUrl: z.string().optional(), + linkTtlSeconds: z.number().int().positive().optional(), + maxUploadBytes: z.number().int().positive().optional(), + }) + .strict() + .optional(); + export const AgentEntrySchema = z .object({ id: z.string(), @@ -680,6 +745,8 @@ export const AgentEntrySchema = z model: AgentModelSchema.optional(), skills: z.array(z.string()).optional(), memorySearch: MemorySearchSchema, + deepMemory: DeepMemorySchema, + rustfs: RustFsSchema, humanDelay: HumanDelaySchema.optional(), heartbeat: HeartbeatSchema, identity: IdentitySchema, diff --git a/src/deep-memory/client.ts b/src/deep-memory/client.ts new file mode 100644 index 0000000000000..493cabc1d8c14 --- /dev/null +++ b/src/deep-memory/client.ts @@ -0,0 +1,282 @@ +import { createSubsystemLogger } from "../logging/subsystem.js"; + +const log = createSubsystemLogger("deep-memory"); + +export type DeepMemoryRetrieveResponse = { + context?: string; + entities?: string[]; + topics?: string[]; + memories?: Array<{ + id?: string; + content?: string; + importance?: number; + relevance?: number; + }>; +}; + +export type DeepMemoryUpdateResponse = { + status?: string; + memories_added?: number; + memories_filtered?: number; +}; + +export type DeepMemoryInspectSessionResponse = { + namespace?: string; + session_id?: string; + topics?: Array<{ name?: string; frequency?: number }>; + entities?: Array<{ name?: string; frequency?: number }>; + summary?: string; + memories?: Array<{ + id?: string; + importance?: number; + created_at?: string; + content?: string; + topics?: string[]; + entities?: string[]; + }>; +}; + +export type DeepMemoryForgetResponse = { + status?: string; + namespace?: string; + deleted?: number; + delete_ids?: number; + delete_session?: number; + error?: string; + // oxlint-disable-next-line typescript/no-explicit-any + details?: any; +}; + +export type DeepMemoryHealthResponse = Record; + +type CacheEntry = { value: T; expiresAt: number }; + +export class DeepMemoryClient { + private readonly baseUrl: string; + private readonly timeoutMs: number; + private readonly cacheEnabled: boolean; + private readonly cacheTtlMs: number; + private readonly cacheMaxEntries: number; + private readonly cache = new Map>(); + private readonly namespace?: string; + private readonly apiKey?: string; + + constructor(params: { + baseUrl: string; + timeoutMs: number; + cache: { enabled: boolean; ttlMs: number; maxEntries: number }; + namespace?: string; + apiKey?: string; + }) { + this.baseUrl = params.baseUrl.replace(/\/+$/, ""); + this.timeoutMs = params.timeoutMs; + this.cacheEnabled = params.cache.enabled; + this.cacheTtlMs = params.cache.ttlMs; + this.cacheMaxEntries = params.cache.maxEntries; + this.namespace = params.namespace?.trim() || undefined; + this.apiKey = params.apiKey?.trim() || undefined; + } + + private pruneCache(): void { + if (!this.cacheEnabled) { + return; + } + const now = Date.now(); + for (const [key, entry] of this.cache.entries()) { + if (entry.expiresAt <= now) { + this.cache.delete(key); + } + } + if (this.cache.size <= this.cacheMaxEntries) { + return; + } + // Best-effort eviction: drop oldest (Map insertion order). + const overflow = this.cache.size - this.cacheMaxEntries; + for (let i = 0; i < overflow; i += 1) { + const first = this.cache.keys().next().value; + if (!first) { + break; + } + this.cache.delete(first); + } + } + + private async postJson(path: string, body: unknown): Promise { + const url = `${this.baseUrl}${path.startsWith("/") ? path : `/${path}`}`; + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), this.timeoutMs); + try { + const res = await fetch(url, { + method: "POST", + headers: { + "content-type": "application/json", + ...(this.apiKey ? { "x-api-key": this.apiKey } : {}), + }, + body: JSON.stringify(body ?? {}), + signal: ctrl.signal, + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + throw new Error(`HTTP ${res.status} ${res.statusText}${text ? `: ${text}` : ""}`); + } + return (await res.json()) as T; + } finally { + clearTimeout(timer); + } + } + + private async getJson(path: string): Promise { + const url = `${this.baseUrl}${path.startsWith("/") ? path : `/${path}`}`; + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), this.timeoutMs); + try { + const headers: Record = {}; + if (this.apiKey) { + headers["x-api-key"] = this.apiKey; + } + const res = await fetch(url, { + method: "GET", + headers, + signal: ctrl.signal, + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + throw new Error(`HTTP ${res.status} ${res.statusText}${text ? `: ${text}` : ""}`); + } + return (await res.json()) as T; + } finally { + clearTimeout(timer); + } + } + + async retrieveContext(params: { + userInput: string; + sessionId: string; + maxMemories: number; + }): Promise { + const input = params.userInput.trim(); + if (!input) { + return {}; + } + const key = `${this.namespace ?? ""}::${params.sessionId}::${params.maxMemories}::${input}`; + if (this.cacheEnabled && this.cacheTtlMs > 0) { + this.pruneCache(); + const cached = this.cache.get(key); + if (cached && cached.expiresAt > Date.now()) { + return cached.value; + } + } + try { + const value = await this.postJson("/retrieve_context", { + namespace: this.namespace, + user_input: input, + session_id: params.sessionId, + max_memories: params.maxMemories, + }); + if (this.cacheEnabled && this.cacheTtlMs > 0) { + this.cache.set(key, { value, expiresAt: Date.now() + this.cacheTtlMs }); + } + return value; + } catch (err) { + log.warn(`retrieve_context failed: ${String(err)}`); + return {}; + } + } + + async inspectSession(params: { + sessionId: string; + limit?: number; + includeContent?: boolean; + }): Promise { + try { + return await this.postJson("/session/inspect", { + namespace: this.namespace, + session_id: params.sessionId, + limit: params.limit, + include_content: params.includeContent, + }); + } catch (err) { + log.warn(`session inspect failed: ${String(err)}`); + return {}; + } + } + + async updateMemoryIndex(params: { + sessionId: string; + messages: unknown[]; + async: boolean; + }): Promise { + try { + return await this.postJson("/update_memory_index", { + namespace: this.namespace, + session_id: params.sessionId, + messages: params.messages, + async: params.async, + }); + } catch (err) { + log.warn(`update_memory_index failed: ${String(err)}`); + return {}; + } + } + + async forget(params: { + sessionId?: string; + memoryIds?: string[]; + dryRun: boolean; + }): Promise<{ ok: true; value: DeepMemoryForgetResponse } | { ok: false; error: string }> { + try { + const value = await this.postJson("/forget", { + namespace: this.namespace, + session_id: params.sessionId, + memory_ids: params.memoryIds, + dry_run: params.dryRun, + }); + return { ok: true, value }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + log.warn(`forget failed: ${message}`); + return { ok: false, error: message }; + } + } + + async health(params?: { + details?: boolean; + }): Promise<{ ok: true; value: DeepMemoryHealthResponse } | { ok: false; error: string }> { + const details = params?.details ?? false; + const path = details ? "/health/details" : "/health"; + try { + const value = await this.getJson(path); + return { ok: true, value }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + log.warn(`health failed: ${message}`); + return { ok: false, error: message }; + } + } + + async readyz(): Promise< + { ok: true; value: Record } | { ok: false; error: string } + > { + try { + const value = await this.getJson>("/readyz"); + return { ok: true, value }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + log.warn(`readyz failed: ${message}`); + return { ok: false, error: message }; + } + } + + async queueStats(): Promise< + { ok: true; value: Record } | { ok: false; error: string } + > { + try { + const value = await this.getJson>("/queue/stats"); + return { ok: true, value }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + log.warn(`queue stats failed: ${message}`); + return { ok: false, error: message }; + } + } +} diff --git a/src/deep-memory/update-scheduler.test.ts b/src/deep-memory/update-scheduler.test.ts new file mode 100644 index 0000000000000..2bdbf520a5645 --- /dev/null +++ b/src/deep-memory/update-scheduler.test.ts @@ -0,0 +1,143 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import type { OpenClawConfig } from "../config/config.js"; +import type { SessionEntry } from "../config/sessions.js"; +import { + considerDeepMemoryUpdateForTranscriptDelta, + considerDeepMemoryUpdateNearCompaction, +} from "./update-scheduler.js"; + +function buildCfg(overrides?: Partial): OpenClawConfig { + return { + agents: { + defaults: { + deepMemory: { + enabled: true, + baseUrl: "http://deep-memory.test", + timeoutSeconds: 2, + retrieve: { maxMemories: 10, cache: { enabled: false, ttlMinutes: 0, maxEntries: 10 } }, + inject: { maxChars: 2000, label: "Deep Memory" }, + update: { + enabled: true, + thresholds: { deltaBytes: 1, deltaMessages: 0 }, + debounceMs: 0, + nearCompaction: { enabled: true }, + }, + }, + }, + }, + ...overrides, + }; +} + +async function writeTranscript(tmpFile: string, lines: unknown[]) { + const payload = lines.map((l) => JSON.stringify(l)).join("\n") + "\n"; + await fs.writeFile(tmpFile, payload, "utf-8"); +} + +describe("deep-memory update scheduler", () => { + it("enqueues update on transcript delta threshold (bytes)", async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-deepmem-")); + const sessionFile = path.join(tmpDir, "s.jsonl"); + await writeTranscript(sessionFile, [{ message: { role: "user", content: "hi" } }]); + + const fetchCalls: string[] = []; + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = + typeof input === "string" + ? input + : input instanceof URL + ? input.toString() + : input instanceof Request + ? input.url + : ""; + fetchCalls.push(url); + return new Response(JSON.stringify({ status: "ok" }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }); + + const cfg = buildCfg(); + const sessionEntry: SessionEntry = { + sessionId: "sess-1", + updatedAt: Date.now(), + sessionFile, + deepMemoryTranscriptSize: 0, + deepMemoryTranscriptLines: 0, + }; + + await considerDeepMemoryUpdateForTranscriptDelta({ + cfg, + agentId: "test", + sessionKey: "agent:test:main", + sessionId: "sess-1", + sessionFile, + sessionEntry, + storePath: undefined, + }); + + // Debounce is 0ms; allow microtasks to flush. + await new Promise((r) => setTimeout(r, 10)); + + expect(fetchCalls.some((u) => u.includes("/update_memory_index"))).toBe(true); + }); + + it("enqueues near-compaction update once per compaction cycle", async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-deepmem-")); + const sessionFile = path.join(tmpDir, "s.jsonl"); + await writeTranscript(sessionFile, [{ message: { role: "user", content: "hi" } }]); + + const fetchCalls: string[] = []; + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = + typeof input === "string" + ? input + : input instanceof URL + ? input.toString() + : input instanceof Request + ? input.url + : ""; + fetchCalls.push(url); + return new Response(JSON.stringify({ status: "ok" }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }); + + const cfg = buildCfg(); + + considerDeepMemoryUpdateNearCompaction({ + cfg, + agentId: "test", + sessionKey: "agent:test:main", + sessionId: "sess-1", + sessionFile, + storePath: undefined, + compactionCount: 3, + deepMemoryNearCompactionCount: undefined, + }); + + // allow async to run + await new Promise((r) => setTimeout(r, 10)); + const firstCount = fetchCalls.filter((u) => u.includes("/update_memory_index")).length; + expect(firstCount).toBe(1); + + // same cycle should be ignored by caller-provided marker + considerDeepMemoryUpdateNearCompaction({ + cfg, + agentId: "test", + sessionKey: "agent:test:main", + sessionId: "sess-1", + sessionFile, + storePath: undefined, + compactionCount: 3, + deepMemoryNearCompactionCount: 3, + }); + await new Promise((r) => setTimeout(r, 10)); + const secondCount = fetchCalls.filter((u) => u.includes("/update_memory_index")).length; + expect(secondCount).toBe(1); + }); +}); diff --git a/src/deep-memory/update-scheduler.ts b/src/deep-memory/update-scheduler.ts new file mode 100644 index 0000000000000..3526b1697d4a0 --- /dev/null +++ b/src/deep-memory/update-scheduler.ts @@ -0,0 +1,338 @@ +import fs from "node:fs/promises"; +import { resolveDeepMemoryConfig } from "../agents/deep-memory.js"; +import type { OpenClawConfig } from "../config/config.js"; +import type { SessionEntry } from "../config/sessions.js"; +import { updateSessionStoreEntry } from "../config/sessions.js"; +import { capArrayByJsonBytes, readSessionMessages } from "../gateway/session-utils.fs.js"; +import { createSubsystemLogger } from "../logging/subsystem.js"; +import { DeepMemoryClient } from "./client.js"; + +const log = createSubsystemLogger("deep-memory"); + +const UPDATE_PAYLOAD_MAX_BYTES = 2 * 1024 * 1024; + +type DeltaState = { + lastSize: number; + lastLines: number; + pendingBytes: number; + pendingLines: number; +}; + +const STATE_BY_SESSION_KEY = new Map(); +const IN_FLIGHT_BY_SESSION_KEY = new Map>(); +const DEBOUNCE_TIMER_BY_SESSION_KEY = new Map(); + +async function countNewlines(absPath: string, start: number, end: number): Promise { + if (end <= start) { + return 0; + } + const handle = await fs.open(absPath, "r"); + try { + let offset = start; + let count = 0; + const buf = Buffer.alloc(64 * 1024); + while (offset < end) { + const toRead = Math.min(buf.length, end - offset); + const { bytesRead } = await handle.read(buf, 0, toRead, offset); + if (bytesRead <= 0) { + break; + } + for (let i = 0; i < bytesRead; i += 1) { + if (buf[i] === 10) { + count += 1; + } + } + offset += bytesRead; + } + return count; + } finally { + await handle.close(); + } +} + +async function readTranscriptStat(absPath: string): Promise<{ size: number }> { + try { + const stat = await fs.stat(absPath); + return { size: stat.size }; + } catch { + return { size: 0 }; + } +} + +function getOrInitDeltaState(sessionKey: string, entry?: SessionEntry): DeltaState { + const existing = STATE_BY_SESSION_KEY.get(sessionKey); + if (existing) { + return existing; + } + const state: DeltaState = { + lastSize: entry?.deepMemoryTranscriptSize ?? 0, + lastLines: entry?.deepMemoryTranscriptLines ?? 0, + pendingBytes: 0, + pendingLines: 0, + }; + STATE_BY_SESSION_KEY.set(sessionKey, state); + return state; +} + +async function persistDeltaBaseline(params: { + storePath?: string; + sessionKey?: string; + size: number; + lines: number; +}) { + if (!params.storePath || !params.sessionKey) { + return; + } + try { + await updateSessionStoreEntry({ + storePath: params.storePath, + sessionKey: params.sessionKey, + update: async () => ({ + deepMemoryTranscriptSize: params.size, + deepMemoryTranscriptLines: params.lines, + }), + }); + } catch (err) { + log.debug(`failed to persist deep memory delta baseline: ${String(err)}`); + } +} + +async function persistUpdateMarkers(params: { + storePath?: string; + sessionKey?: string; + now: number; + nearCompactionCount?: number; +}) { + if (!params.storePath || !params.sessionKey) { + return; + } + try { + await updateSessionStoreEntry({ + storePath: params.storePath, + sessionKey: params.sessionKey, + update: async () => ({ + deepMemoryUpdatedAt: params.now, + ...(typeof params.nearCompactionCount === "number" + ? { deepMemoryNearCompactionCount: params.nearCompactionCount } + : {}), + }), + }); + } catch (err) { + log.debug(`failed to persist deep memory update markers: ${String(err)}`); + } +} + +async function runUpdateNow(params: { + cfg: OpenClawConfig; + agentId: string; + sessionKey: string; + sessionId: string; + sessionFile: string; + storePath?: string; + nearCompactionCount?: number; +}) { + const deepCfg = resolveDeepMemoryConfig(params.cfg, params.agentId); + if (!deepCfg || !deepCfg.update.enabled) { + return; + } + const client = new DeepMemoryClient({ + baseUrl: deepCfg.baseUrl, + timeoutMs: deepCfg.timeoutMs, + cache: deepCfg.retrieve.cache, + namespace: deepCfg.namespace, + apiKey: deepCfg.apiKey, + }); + const messages = readSessionMessages(params.sessionId, params.storePath, params.sessionFile); + const bounded = capArrayByJsonBytes(messages, UPDATE_PAYLOAD_MAX_BYTES).items; + await client.updateMemoryIndex({ + sessionId: params.sessionId, + messages: bounded, + async: true, + }); + const now = Date.now(); + await persistUpdateMarkers({ + storePath: params.storePath, + sessionKey: params.sessionKey, + now, + nearCompactionCount: params.nearCompactionCount, + }); +} + +function enqueueDebounced(params: { + cfg: OpenClawConfig; + agentId: string; + sessionKey: string; + sessionId: string; + sessionFile: string; + storePath?: string; + debounceMs: number; + nearCompactionCount?: number; +}) { + const existing = DEBOUNCE_TIMER_BY_SESSION_KEY.get(params.sessionKey); + if (existing) { + clearTimeout(existing); + DEBOUNCE_TIMER_BY_SESSION_KEY.delete(params.sessionKey); + } + const timer = setTimeout( + () => { + DEBOUNCE_TIMER_BY_SESSION_KEY.delete(params.sessionKey); + void enqueueImmediate(params).catch((err) => { + log.warn(`deep memory update failed: ${String(err)}`); + }); + }, + Math.max(0, params.debounceMs), + ); + DEBOUNCE_TIMER_BY_SESSION_KEY.set(params.sessionKey, timer); +} + +async function enqueueImmediate(params: { + cfg: OpenClawConfig; + agentId: string; + sessionKey: string; + sessionId: string; + sessionFile: string; + storePath?: string; + debounceMs: number; + nearCompactionCount?: number; +}) { + const inflight = IN_FLIGHT_BY_SESSION_KEY.get(params.sessionKey); + if (inflight) { + return inflight; + } + const task = runUpdateNow({ + cfg: params.cfg, + agentId: params.agentId, + sessionKey: params.sessionKey, + sessionId: params.sessionId, + sessionFile: params.sessionFile, + storePath: params.storePath, + nearCompactionCount: params.nearCompactionCount, + }).finally(() => { + IN_FLIGHT_BY_SESSION_KEY.delete(params.sessionKey); + }); + IN_FLIGHT_BY_SESSION_KEY.set(params.sessionKey, task); + return task; +} + +/** + * Trigger #2: called after a normal turn completes to batch updates by transcript deltas. + */ +export async function considerDeepMemoryUpdateForTranscriptDelta(params: { + cfg: OpenClawConfig; + agentId: string; + sessionKey?: string; + sessionId: string; + sessionFile: string; + sessionEntry?: SessionEntry; + storePath?: string; +}): Promise { + const sessionKey = params.sessionKey?.trim(); + if (!sessionKey) { + return; + } + const deepCfg = resolveDeepMemoryConfig(params.cfg, params.agentId); + if (!deepCfg || !deepCfg.update.enabled) { + return; + } + const thresholds = deepCfg.update.thresholds; + const bytesThreshold = thresholds.deltaBytes; + const linesThreshold = thresholds.deltaMessages; + if (bytesThreshold <= 0 && linesThreshold <= 0) { + return; + } + const state = getOrInitDeltaState(sessionKey, params.sessionEntry); + const stat = await readTranscriptStat(params.sessionFile); + const size = stat.size; + + const deltaBytes = Math.max(0, size - state.lastSize); + state.pendingBytes += deltaBytes; + + let deltaLines = 0; + if (linesThreshold > 0 && size > state.lastSize) { + deltaLines = await countNewlines(params.sessionFile, state.lastSize, size); + state.pendingLines += deltaLines; + } + + state.lastSize = size; + state.lastLines += deltaLines; + + // Persist baseline so restarts continue from the right offset. + await persistDeltaBaseline({ + storePath: params.storePath, + sessionKey, + size: state.lastSize, + lines: state.lastLines, + }); + + const hitBytes = + bytesThreshold > 0 ? state.pendingBytes >= bytesThreshold : state.pendingBytes > 0; + const hitLines = + linesThreshold > 0 ? state.pendingLines >= linesThreshold : state.pendingLines > 0; + if (!hitBytes && !hitLines) { + return; + } + + // Consume thresholds (best-effort) so we don't spam updates if updates are slow. + if (bytesThreshold > 0) { + state.pendingBytes = Math.max(0, state.pendingBytes - bytesThreshold); + } else { + state.pendingBytes = 0; + } + if (linesThreshold > 0) { + state.pendingLines = Math.max(0, state.pendingLines - linesThreshold); + } else { + state.pendingLines = 0; + } + + enqueueDebounced({ + cfg: params.cfg, + agentId: params.agentId, + sessionKey, + sessionId: params.sessionId, + sessionFile: params.sessionFile, + storePath: params.storePath, + debounceMs: deepCfg.update.debounceMs, + }); +} + +/** + * Trigger #3: called when a session is nearing auto-compaction. + * Enqueues an update once per compaction cycle (best-effort). + */ +export function considerDeepMemoryUpdateNearCompaction(params: { + cfg: OpenClawConfig; + agentId: string; + sessionKey?: string; + sessionId: string; + sessionFile: string; + storePath?: string; + compactionCount?: number; + deepMemoryNearCompactionCount?: number; +}) { + const sessionKey = params.sessionKey?.trim(); + if (!sessionKey) { + return; + } + const deepCfg = resolveDeepMemoryConfig(params.cfg, params.agentId); + if (!deepCfg || !deepCfg.update.enabled || !deepCfg.update.nearCompaction.enabled) { + return; + } + const cycle = params.compactionCount ?? 0; + const last = params.deepMemoryNearCompactionCount; + if (typeof last === "number" && last === cycle) { + return; + } + // No debounce here: this is a safety/critical trigger (but still async). + void enqueueImmediate({ + cfg: params.cfg, + agentId: params.agentId, + sessionKey, + sessionId: params.sessionId, + sessionFile: params.sessionFile, + storePath: params.storePath, + debounceMs: 0, + nearCompactionCount: cycle, + }).catch((err) => { + log.warn(`deep memory near-compaction update failed: ${String(err)}`); + }); +} diff --git a/src/rustfs/client.ts b/src/rustfs/client.ts new file mode 100644 index 0000000000000..5e12ee10abe3f --- /dev/null +++ b/src/rustfs/client.ts @@ -0,0 +1,265 @@ +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { Readable } from "node:stream"; + +export type RustFsFileMeta = { + file_id: string; + tenant_id: string; + session_id?: string; + filename: string; + mime?: string; + size: number; + sha256: string; + created_at_ms: number; + source?: string; + encrypted: boolean; + extract_status?: string; + extract_updated_at_ms?: number; + extract_attempt?: number; + extract_error?: string; + annotations?: unknown; +}; + +export type RustFsSearchResponse = + | { ok: true; items: RustFsFileMeta[] } + | { ok: false; error: string }; + +export type RustFsIngestResponse = + | { ok: true; file_id: string; sha256: string; size: number; encrypted: boolean } + | { ok: false; error: string }; + +export type RustFsLinkResponse = + | { ok: true; token: string; path: string; url?: string; expires_at_ms: number } + | { ok: false; error: string }; + +export type RustFsAnnotationsResponse = + | { ok: true; file_id: string; updated_at_ms: number } + | { ok: false; error: string }; + +export class RustFsClient { + private readonly baseUrl: string; + private readonly apiKey?: string; + private readonly project: string; + private readonly linkTtlSeconds: number; + private readonly maxUploadBytes: number; + + constructor(params: { + baseUrl: string; + apiKey?: string; + project: string; + linkTtlSeconds: number; + maxUploadBytes: number; + }) { + this.baseUrl = params.baseUrl.replace(/\/+$/, ""); + this.apiKey = params.apiKey?.trim() || undefined; + this.project = params.project.trim(); + this.linkTtlSeconds = params.linkTtlSeconds; + this.maxUploadBytes = params.maxUploadBytes; + } + + private headers(extra?: Record): Record { + return { + ...(this.apiKey ? { "x-api-key": this.apiKey } : undefined), + ...extra, + }; + } + + async search(params: { + query?: string; + sessionId?: string; + mime?: string; + extractStatus?: string; + limit?: number; + }): Promise { + const qs = new URLSearchParams(); + qs.set("tenant_id", this.project); + if (params.query?.trim()) { + qs.set("q", params.query.trim()); + } + if (params.sessionId?.trim()) { + qs.set("session_id", params.sessionId.trim()); + } + if (params.mime?.trim()) { + qs.set("mime", params.mime.trim()); + } + if (params.extractStatus?.trim()) { + qs.set("extract_status", params.extractStatus.trim()); + } + if (typeof params.limit === "number" && Number.isFinite(params.limit)) { + qs.set("limit", String(Math.max(1, Math.min(200, Math.trunc(params.limit))))); + } + const url = `${this.baseUrl}/v1/files?${qs.toString()}`; + try { + const res = await fetch(url, { headers: this.headers() }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + return { + ok: false, + error: `HTTP ${res.status} ${res.statusText}${text ? `: ${text}` : ""}`, + }; + } + const json = (await res.json()) as { ok?: unknown; items?: unknown }; + const items = Array.isArray(json.items) ? (json.items as RustFsFileMeta[]) : []; + return { ok: true, items }; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : String(err) }; + } + } + + async createLink(params: { fileId: string; ttlSeconds?: number }): Promise { + const fileId = params.fileId.trim(); + if (!fileId) { + return { ok: false, error: "fileId required" }; + } + const ttlSeconds = + typeof params.ttlSeconds === "number" && Number.isFinite(params.ttlSeconds) + ? Math.max(30, Math.min(3600, Math.trunc(params.ttlSeconds))) + : this.linkTtlSeconds; + try { + const res = await fetch(`${this.baseUrl}/v1/files/${encodeURIComponent(fileId)}/link`, { + method: "POST", + headers: this.headers({ "content-type": "application/json" }), + body: JSON.stringify({ ttl_seconds: ttlSeconds }), + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + return { + ok: false, + error: `HTTP ${res.status} ${res.statusText}${text ? `: ${text}` : ""}`, + }; + } + const json = (await res.json()) as Record; + if (json.ok !== true) { + return { ok: false, error: "unexpected response" }; + } + return json as unknown as RustFsLinkResponse; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : String(err) }; + } + } + + async ingestFile(params: { + absPath: string; + sessionId?: string; + source?: string; + filename?: string; + mime?: string; + }): Promise { + const absPath = params.absPath.trim(); + if (!absPath) { + return { ok: false, error: "path required" }; + } + const stat = await fs.promises.stat(absPath).catch(() => null); + if (!stat || !stat.isFile()) { + return { ok: false, error: "file not found" }; + } + if (stat.size > this.maxUploadBytes) { + return { ok: false, error: `file too large (${stat.size} bytes)` }; + } + const filename = (params.filename ?? path.basename(absPath)).trim() || path.basename(absPath); + const mime = params.mime?.trim() || "application/octet-stream"; + + // Stream multipart upload without buffering file in memory. + const boundary = `----openclaw-rustfs-${crypto.randomUUID()}`; + const fileStream = fs.createReadStream(absPath); + + const headerField = (name: string, value: string) => + Buffer.from( + `--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\n\r\n${value}\r\n`, + "utf8", + ); + const fileHeader = Buffer.from( + `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${filename.replaceAll( + '"', + "_", + )}"\r\nContent-Type: ${mime}\r\n\r\n`, + "utf8", + ); + const footer = Buffer.from(`\r\n--${boundary}--\r\n`, "utf8"); + const tenantId = this.project; + + const body = Readable.from( + (async function* () { + yield headerField("tenant_id", tenantId); + if (params.sessionId?.trim()) { + yield headerField("session_id", params.sessionId.trim()); + } + if (params.source?.trim()) { + yield headerField("source", params.source.trim()); + } + yield fileHeader; + for await (const chunk of fileStream) { + yield chunk as Buffer; + } + yield footer; + })(), + ); + const webBody = Readable.toWeb(body) as ReadableStream; + + const url = `${this.baseUrl}/v1/files`; + try { + const res = await fetch(url, { + method: "POST", + headers: this.headers({ + "content-type": `multipart/form-data; boundary=${boundary}`, + }), + body: webBody, + duplex: "half", + } as RequestInit & { duplex: "half" }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + return { + ok: false, + error: `HTTP ${res.status} ${res.statusText}${text ? `: ${text}` : ""}`, + }; + } + const json = (await res.json()) as Record; + if (json.ok !== true) { + return { ok: false, error: "unexpected response" }; + } + return json as unknown as RustFsIngestResponse; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : String(err) }; + } + } + + async upsertAnnotations(params: { + fileId: string; + // Opaque: semantics are LLM/worker-defined. + annotations: unknown; + source?: string; + }): Promise { + const fileId = params.fileId.trim(); + if (!fileId) { + return { ok: false, error: "fileId required" }; + } + try { + const res = await fetch( + `${this.baseUrl}/v1/files/${encodeURIComponent(fileId)}/annotations`, + { + method: "POST", + headers: this.headers({ "content-type": "application/json" }), + body: JSON.stringify({ + annotations: params.annotations ?? {}, + source: params.source?.trim() || "openclaw", + }), + }, + ); + if (!res.ok) { + const text = await res.text().catch(() => ""); + return { + ok: false, + error: `HTTP ${res.status} ${res.statusText}${text ? `: ${text}` : ""}`, + }; + } + const json = (await res.json()) as Record; + if (json.ok !== true) { + return { ok: false, error: "unexpected response" }; + } + return json as unknown as RustFsAnnotationsResponse; + } catch (err) { + return { ok: false, error: err instanceof Error ? err.message : String(err) }; + } + } +}