unified_server.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
GödelOS Unified Backend Server

A consolidated server that combines the stability of the minimal server
with the advanced cognitive capabilities of the main server.
This server provides complete functionality with reliable dependencies.
"""

import asyncio
import json
import logging
import os
import sys
import time
import uuid
from contextlib import asynccontextmanager
from datetime import datetime
from typing import Dict, List, Optional, Any, Union

import uvicorn
from fastapi import FastAPI, HTTPException, WebSocket, WebSocketDisconnect, File, UploadFile, Form, Query, Header, Depends
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse, HTMLResponse, Response
from pydantic import BaseModel
from dotenv import load_dotenv

# Ensure repository root is on sys.path before importing backend.* packages
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from backend.core.errors import CognitiveError, from_exception
from backend.schemas import (
    WikipediaImportSchema,
    URLImportSchema,
    TextImportSchema,
    BatchImportSchema,
    AddKnowledgeSchema,
    EnhancedCognitiveQuerySchema,
)
from backend.core.structured_logging import (
    setup_structured_logging, correlation_context, CorrelationTracker,
    api_logger, performance_logger, track_operation
)
from backend.core.enhanced_metrics import metrics_collector, operation_timer, collect_metrics

# Load environment variables from .env file
load_dotenv()

# Setup enhanced logging
setup_structured_logging(
    log_level=os.getenv("LOG_LEVEL", "INFO"),
    log_file=os.getenv("LOG_FILE"),
    enable_json=os.getenv("ENABLE_JSON_LOGGING", "true").lower() == "true",
    enable_console=True
)
logger = logging.getLogger(__name__)

# (PYTHONPATH insertion is done above, before importing backend.*)


def _structured_http_error(status: int, *, code: str, message: str, recoverable: bool = False, service: Optional[str] = None, **details) -> HTTPException:
    """Create a standardized HTTPException detail using CognitiveError."""
    err = CognitiveError(code=code, message=message, recoverable=recoverable, details={**({"service": service} if service else {}), **details})
    return HTTPException(status_code=status, detail=err.to_dict())

# Core model definitions
class QueryRequest(BaseModel):
    query: str
    context: Optional[Dict[str, Any]] = None
    stream: Optional[bool] = False

class QueryResponse(BaseModel):
    response: str
    confidence: Optional[float] = None
    reasoning_trace: Optional[List[str]] = None
    sources: Optional[List[str]] = None
    inference_time_ms: Optional[float] = None
    knowledge_used: Optional[List[str]] = None

class KnowledgeRequest(BaseModel):
    content: str
    source: Optional[str] = None
    metadata: Optional[Dict[str, Any]] = None

class CognitiveStreamConfig(BaseModel):
    enable_reasoning_trace: bool = True
    enable_transparency: bool = True
    stream_interval: int = 1000

class ChatMessage(BaseModel):
    message: str
    context: Optional[Dict[str, Any]] = None

class ChatResponse(BaseModel):
    response: str
    tool_calls: Optional[List[Dict[str, Any]]] = None
    reasoning: Optional[List[str]] = None

# Import GödelOS components - with fallback handling for reliability
try:
    from backend.godelos_integration import GödelOSIntegration
    GODELOS_AVAILABLE = True
except ImportError as e:
    logger.warning(f"GödelOS integration not available: {e}")
    GödelOSIntegration = None
    GODELOS_AVAILABLE = False

# Use unified WebSocket manager (no external dependency)
class WebSocketManager:
    def __init__(self):
        self.active_connections: List[WebSocket] = []
    
    async def connect(self, websocket: WebSocket):
        await websocket.accept()
        self.active_connections.append(websocket)
    
    def disconnect(self, websocket: WebSocket):
        if websocket in self.active_connections:
            self.active_connections.remove(websocket)
    
    async def send_personal_message(self, message: str, websocket: WebSocket):
        await websocket.send_text(message)
    
    async def broadcast(self, message: Union[str, dict]):
        if isinstance(message, dict):
            message = json.dumps(message)
        for connection in self.active_connections:
            try:
                await connection.send_text(message)
            except:
                pass  # Connection closed
    
    async def broadcast_cognitive_update(self, event: dict):
        """Broadcast cognitive update event to all connected clients"""
        # Allow callers to send either a raw event dict or an already-wrapped
        # { type: 'cognitive_event', data: {...} } message. Normalize to raw event.
        try:
            inner_event = event
            if isinstance(event, dict) and event.get("type") == "cognitive_event" and isinstance(event.get("data"), dict):
                inner_event = event.get("data")
            message = {
                "type": "cognitive_event",
                "timestamp": inner_event.get("timestamp", ""),
                "data": inner_event
            }
        except Exception:
            # Fallback if anything unexpected
            message = {
                "type": "cognitive_event",
                "timestamp": event.get("timestamp", ""),
                "data": event
            }
        await self.broadcast(message)
    
    async def broadcast_consciousness_update(self, consciousness_data: dict):
        """Broadcast consciousness update to all connected clients"""
        try:
            message = {
                "type": "consciousness_update",
                "timestamp": consciousness_data.get("timestamp", time.time()),
                "data": consciousness_data
            }
            await self.broadcast(message)
        except Exception as e:
            logger.error(f"Error broadcasting consciousness update: {e}")
    
    def has_connections(self) -> bool:
        return len(self.active_connections) > 0

WEBSOCKET_MANAGER_AVAILABLE = True

# Import LLM tool integration

try:
    from backend.llm_tool_integration import ToolBasedLLMIntegration
    LLM_INTEGRATION_AVAILABLE = True
except ImportError as e:
    logger.warning(f"LLM integration not available: {e}")
    # Create a mock LLM integration for basic functionality
    class MockToolBasedLLMIntegration:
        def __init__(self, godelos_integration):
            self.godelos_integration = godelos_integration
            self.tools = []
        
        async def test_integration(self):
            return {"test_successful": True, "tool_calls": 0}
        
        async def process_query(self, query):
            return {
                "response": f"Processing query: '{query}' - Basic cognitive processing active (mock LLM mode)",
                "confidence": 0.8,
                "reasoning_trace": ["Query received", "Basic processing applied", "Response generated"],
                "sources": ["internal_reasoning"]
            }
    
    ToolBasedLLMIntegration = MockToolBasedLLMIntegration
    LLM_INTEGRATION_AVAILABLE = True

# Import LLM cognitive driver for consciousness assessment
try:
    from backend.llm_cognitive_driver import LLMCognitiveDriver
    LLM_COGNITIVE_DRIVER_AVAILABLE = True
except ImportError as e:
    logger.warning(f"LLM cognitive driver not available: {e}")
    LLM_COGNITIVE_DRIVER_AVAILABLE = False

# Import additional services with fallbacks
# Import each service independently so a failure in one (e.g. thinc/spaCy for
# knowledge_management) doesn't take down the ingestion service.
knowledge_ingestion_service = None
knowledge_management_service = None
knowledge_pipeline_service = None
KNOWLEDGE_SERVICES_AVAILABLE = False

try:
    from backend.knowledge_ingestion import knowledge_ingestion_service as _kis
    knowledge_ingestion_service = _kis
    KNOWLEDGE_SERVICES_AVAILABLE = True
except ImportError as e:
    logger.warning(f"Knowledge ingestion service not available: {e}")

try:
    from backend.knowledge_management import knowledge_management_service as _kms
    knowledge_management_service = _kms
except ImportError as e:
    logger.warning(f"Knowledge management service not available: {e}")

try:
    from backend.knowledge_pipeline_service import knowledge_pipeline_service as _kps
    knowledge_pipeline_service = _kps
except ImportError as e:
    logger.warning(f"Knowledge pipeline service not available: {e}")

# Import production vector database
try:
    from backend.core.vector_service import get_vector_database, init_vector_database
    from backend.core.vector_endpoints import router as vector_db_router
    VECTOR_DATABASE_AVAILABLE = True
    logger.info("Production vector database available")
except ImportError as e:
    logger.warning(f"Production vector database not available, using fallback: {e}")
    get_vector_database = None
    init_vector_database = None
    vector_db_router = None
    VECTOR_DATABASE_AVAILABLE = False

# Import distributed vector database
try:
    from backend.api.distributed_vector_router import router as distributed_vector_router
    DISTRIBUTED_VECTOR_AVAILABLE = True
    logger.info("Distributed vector database available")
except ImportError as e:
    logger.warning(f"Distributed vector database not available: {e}")
    distributed_vector_router = None
    DISTRIBUTED_VECTOR_AVAILABLE = False

try:
    from backend.enhanced_cognitive_api import router as enhanced_cognitive_router
    from backend.transparency_endpoints import router as transparency_router, initialize_transparency_system
    ENHANCED_APIS_AVAILABLE = True
except ImportError as e:
    logger.warning(f"Enhanced APIs not available: {e}")
    enhanced_cognitive_router = None
    transparency_router = None
    ENHANCED_APIS_AVAILABLE = False

# Import consciousness engine and cognitive manager
try:
    from backend.core.consciousness_engine import ConsciousnessEngine
    from backend.core.cognitive_manager import CognitiveManager
    from backend.core.cognitive_transparency import transparency_engine, initialize_transparency_engine
    CONSCIOUSNESS_AVAILABLE = True
except ImportError as e:
    logger.warning(f"Consciousness engine not available: {e}")
    ConsciousnessEngine = None
    CognitiveManager = None
    CONSCIOUSNESS_AVAILABLE = False

# Import unified consciousness engine
try:
    from backend.core.unified_consciousness_engine import UnifiedConsciousnessEngine
    from backend.core.enhanced_websocket_manager import EnhancedWebSocketManager
    UNIFIED_CONSCIOUSNESS_AVAILABLE = True
    logger.info("✅ Unified consciousness engine available")
except ImportError as e:
    logger.warning(f"Unified consciousness engine not available: {e}")
    UnifiedConsciousnessEngine = None
    EnhancedWebSocketManager = None
    UNIFIED_CONSCIOUSNESS_AVAILABLE = False

# Import dormant module manager
try:
    from backend.core.dormant_module_manager import DormantModuleManager
    DORMANT_MODULE_MANAGER_AVAILABLE = True
except ImportError as e:
    logger.warning(f"DormantModuleManager not available: {e}")
    DormantModuleManager = None
    DORMANT_MODULE_MANAGER_AVAILABLE = False

# Global service instances - using Any to avoid type annotation issues
godelos_integration = None
websocket_manager = None
enhanced_websocket_manager = None
unified_consciousness_engine = None
tool_based_llm = None
cognitive_manager = None
dormant_module_manager = None
self_modification_engine = None
# Removed cognitive_streaming_task - no longer using synthetic streaming

# Observability instances
correlation_tracker = CorrelationTracker()

# Health normalization (single source of truth)
def score_to_label(score: Optional[float]) -> str:
    """Convert numeric health score (0.0-1.0) to categorical label."""
    if score is None:
        return "unknown"
    # Handle NaN values
    if isinstance(score, float) and (score != score):  # NaN check
        return "unknown"
    if score >= 0.8:
        return "healthy"
    if score >= 0.4:
        return "degraded"
    return "down"

def get_system_health_with_labels() -> Dict[str, Any]:
    """Get system health with both numeric values and derived labels."""
    # Get actual health scores from components
    health_scores = {
        "websocketConnection": 1.0 if websocket_manager and len(websocket_manager.active_connections) > 0 else 0.0,
        "pipeline": 0.85,  # Mock value, should come from pipeline service
        "knowledgeStore": 0.92,  # Mock value, should come from knowledge store
        "vectorIndex": 0.88,  # Mock value, should come from vector index
    }
    
    # Compute labels from scores
    labels = {key: score_to_label(value) for key, value in health_scores.items()}
    
    return {
        **health_scores,
        "_labels": labels
    }

def get_manifest_consciousness_canonical() -> Dict[str, Any]:
    """Get manifest consciousness in canonical camelCase format."""
    return {
        "attention": {
            "intensity": 0.7,
            "focus": ["System monitoring"],
            "coverage": 0.85
        },
        "awareness": {
            "level": 0.8,
            "breadth": 0.75
        },
        "metaReflection": {
            "depth": 0.6,
            "coherence": 0.85
        },
        "processMonitoring": {
            "latency": 150.0,  # ms
            "throughput": 0.9
        }
    }

def get_knowledge_stats() -> Dict[str, Any]:
    """Get knowledge statistics."""
    return {
        "totalConcepts": 0,
        "totalConnections": 0,
        "totalDocuments": 0
    }

# Simulated cognitive state for fallback (legacy format)
cognitive_state = {
    "processing_load": 0.65,
    "active_queries": 0,
    "attention_focus": {
        "primary": "System monitoring",
        "secondary": ["Background processing", "Memory consolidation"],
        "intensity": 0.7
    },
    "working_memory": {
        "capacity": 7,
        "current_items": 3,
        "items": ["Query processing", "Knowledge retrieval", "Response generation"]
    },
    "metacognitive_status": {
        "self_awareness": 0.8,
        "confidence": 0.75,
        "uncertainty": 0.25,
        "learning_rate": 0.6
    }
}

async def initialize_core_services():
    """Initialize core services with proper error handling."""
    global godelos_integration, websocket_manager, enhanced_websocket_manager, unified_consciousness_engine, tool_based_llm, cognitive_manager, transparency_engine
    
    # Initialize WebSocket manager
    websocket_manager = WebSocketManager()
    logger.info("✅ WebSocket manager initialized")
    
    # Initialize enhanced WebSocket manager for consciousness streaming
    if UNIFIED_CONSCIOUSNESS_AVAILABLE:
        try:
            enhanced_websocket_manager = EnhancedWebSocketManager()
            logger.info("✅ Enhanced WebSocket manager initialized for consciousness streaming")
        except Exception as e:
            logger.error(f"❌ Failed to initialize enhanced WebSocket manager: {e}")
            enhanced_websocket_manager = websocket_manager  # Fallback to basic manager
    else:
        enhanced_websocket_manager = websocket_manager
    
    # Initialize transparency engine with websocket manager
    transparency_engine = initialize_transparency_engine(enhanced_websocket_manager)
    logger.info("✅ Cognitive transparency engine initialized with WebSocket integration")
    
    # Initialize GödelOS integration if available
    if GODELOS_AVAILABLE:
        try:
            godelos_integration = GödelOSIntegration()
            await godelos_integration.initialize()
            logger.info("✅ GödelOS integration initialized successfully")
        except Exception as e:
            logger.error(f"❌ Failed to initialize GödelOS integration: {e}")
            godelos_integration = None
    
    # Initialize LLM tool integration if available
    if LLM_INTEGRATION_AVAILABLE:
        try:
            tool_based_llm = ToolBasedLLMIntegration(godelos_integration)
            test_result = await tool_based_llm.test_integration()
            if test_result.get("test_successful", False):
                logger.info(f"✅ Tool-based LLM integration initialized - {test_result.get('tool_calls', 0)} tools available")
            else:
                logger.warning("⚠️ Tool-based LLM integration test failed, but system is operational")
        except Exception as e:
            logger.error(f"❌ Failed to initialize LLM integration: {e}")
            tool_based_llm = None
    
    # Initialize LLM cognitive driver for consciousness assessment
    llm_cognitive_driver = None
    if LLM_COGNITIVE_DRIVER_AVAILABLE:
        try:
            llm_cognitive_driver = LLMCognitiveDriver()
            logger.info("✅ LLM cognitive driver initialized for consciousness assessment")
        except Exception as e:
            logger.error(f"❌ Failed to initialize LLM cognitive driver: {e}")
            llm_cognitive_driver = None
    
    # Initialize cognitive manager with consciousness engine if available
    if CONSCIOUSNESS_AVAILABLE and (llm_cognitive_driver or tool_based_llm):
        try:
            # Use LLM cognitive driver for consciousness if available, otherwise fall back to tool-based LLM
            llm_driver_for_consciousness = llm_cognitive_driver if llm_cognitive_driver else tool_based_llm
            
            # Correct argument order: (godelos_integration, llm_driver, knowledge_pipeline, websocket_manager)
            cognitive_manager = CognitiveManager(
                godelos_integration=godelos_integration,
                llm_driver=llm_driver_for_consciousness,
                knowledge_pipeline=None,
                websocket_manager=enhanced_websocket_manager,
            )
            await cognitive_manager.initialize()
            driver_type = "LLM cognitive driver" if llm_cognitive_driver else "tool-based LLM"
            logger.info(f"✅ Cognitive manager with consciousness engine initialized successfully using {driver_type}")

            # Bootstrap consciousness after initialization
            if hasattr(cognitive_manager, 'consciousness_engine') and cognitive_manager.consciousness_engine:
                try:
                    ce = cognitive_manager.consciousness_engine
                    
                    if not ce.is_bootstrap_complete():
                        logger.info("🌅 Bootstrapping consciousness in cognitive manager...")
                        await ce.bootstrap_consciousness()
                        logger.info("✅ Consciousness engine bootstrapped successfully")
                    else:
                        logger.info("🟡 Consciousness engine bootstrap already completed; skipping duplicate call.")
                except Exception as bootstrap_error:
                    logger.warning(f"⚠️ Consciousness bootstrap warning (non-fatal): {bootstrap_error}")

            # Update replay endpoints with cognitive manager
            try:
                from backend.api.replay_endpoints import setup_replay_endpoints
                setup_replay_endpoints(app, cognitive_manager)
                logger.info("✅ Replay endpoints updated with cognitive manager")
            except Exception as e:
                logger.warning(f"Failed to update replay endpoints: {e}")
                
        except Exception as e:
            logger.error(f"❌ Failed to initialize cognitive manager: {e}")
            cognitive_manager = None

    # Initialize unified consciousness engine if available
    if UNIFIED_CONSCIOUSNESS_AVAILABLE:
        try:
            # Use the enhanced websocket manager and LLM driver
            llm_driver_for_consciousness = llm_cognitive_driver if llm_cognitive_driver else tool_based_llm
            
            unified_consciousness_engine = UnifiedConsciousnessEngine(
                websocket_manager=enhanced_websocket_manager,
                llm_driver=llm_driver_for_consciousness
            )
            
            await unified_consciousness_engine.initialize_components()
            logger.info("✅ Unified consciousness engine initialized successfully")
            
            # Set the consciousness engine reference in the enhanced websocket manager for real-time data
            if hasattr(enhanced_websocket_manager, 'set_consciousness_engine'):
                enhanced_websocket_manager.set_consciousness_engine(unified_consciousness_engine)
            
            # Start the consciousness loop
            await unified_consciousness_engine.start_consciousness_loop()
            logger.info("🧠 Unified consciousness loop started")
            
            # Note: Consciousness bootstrap is handled in cognitive_manager initialization above
            # to avoid duplicate bootstrap calls

        except Exception as e:
            logger.error(f"❌ Failed to initialize unified consciousness engine: {e}")
            unified_consciousness_engine = None

async def initialize_optional_services():
    """Initialize optional advanced services."""
    global godelos_integration
    
    # Initialize knowledge services if available
    if KNOWLEDGE_SERVICES_AVAILABLE and knowledge_ingestion_service and knowledge_management_service:
        try:
            # Initialize knowledge ingestion service with websocket manager
            logger.info(f"🔍 UNIFIED_SERVER: Initializing knowledge_ingestion_service with websocket_manager: {websocket_manager is not None}")
            await knowledge_ingestion_service.initialize(websocket_manager)
            await knowledge_management_service.initialize()
            if knowledge_pipeline_service and websocket_manager:
                await knowledge_pipeline_service.initialize(websocket_manager)
                # Wire into cognitive manager if available
                if cognitive_manager is not None:
                    cognitive_manager.knowledge_pipeline = knowledge_pipeline_service
            logger.info("✅ Knowledge services initialized successfully")
        except Exception as e:
            logger.error(f"❌ Failed to initialize knowledge services: {e}")
    
    # Initialize production vector database (synchronous initialization)
    if VECTOR_DATABASE_AVAILABLE:
        try:
            # Use ThreadPoolExecutor with timeout for resilient model initialization
            import asyncio
            from concurrent.futures import ThreadPoolExecutor, TimeoutError
            
            def _init_vector_db():
                """Initialize vector database in thread."""
                if init_vector_database:
                    init_vector_database()
                elif get_vector_database:
                    get_vector_database()
                
            # Initialize with timeout to avoid hanging on model downloads
            loop = asyncio.get_event_loop()
            with ThreadPoolExecutor(max_workers=1) as executor:
                try:
                    await asyncio.wait_for(
                        loop.run_in_executor(executor, _init_vector_db),
                        timeout=30.0  # 30 second timeout
                    )
                    logger.info("✅ Production vector database initialized successfully!")
                except asyncio.TimeoutError:
                    logger.warning("⚠️ Vector database initialization timed out - will retry on demand")
                except Exception as e:
                    logger.warning(f"⚠️ Vector database initialization failed: {e} - will retry on demand")

            # Wire telemetry notifier for vector DB recoverable errors
            try:
                from backend.core.vector_service import set_telemetry_notifier
                if websocket_manager is not None:
                    def _notify(event: dict):
                        # Schedule async broadcast without blocking
                        try:
                            if websocket_manager:
                                asyncio.create_task(websocket_manager.broadcast_cognitive_update(event))
                        except Exception:
                            pass
                    set_telemetry_notifier(_notify)
                    logger.info("✅ Vector DB telemetry notifier wired to WebSocket manager")
            except Exception as e:
                logger.warning(f"Could not wire Vector DB telemetry notifier: {e}")
        except Exception as e:
            logger.error(f"❌ Failed to initialize vector database: {e}")
            import traceback
            logger.error(f"❌ Detailed error: {traceback.format_exc()}")

    # Initialize cognitive transparency API - CRITICAL FOR UNIFIED KG!
    if ENHANCED_APIS_AVAILABLE:
        try:
            from backend.cognitive_transparency_integration import cognitive_transparency_api
            
            # Initialize the cognitive transparency API with GödelOS integration
            logger.info("🔍 UNIFIED_SERVER: Initializing cognitive transparency API for unified KG...")
            await cognitive_transparency_api.initialize(godelos_integration)
            logger.info("✅ Cognitive transparency API initialized successfully - unified KG is ready!")
            
            # Also initialize the transparency system
            if initialize_transparency_system:
                await initialize_transparency_system()
                logger.info("✅ Transparency system initialized successfully")
        except Exception as e:
            logger.error(f"❌ Failed to initialize cognitive transparency system: {e}")
            # Log more details about the failure
            import traceback
            logger.error(f"❌ Detailed error: {traceback.format_exc()}")

    # Initialize dormant module manager
    global dormant_module_manager
    if DORMANT_MODULE_MANAGER_AVAILABLE and DormantModuleManager is not None and godelos_integration is not None:
        try:
            dormant_module_manager = DormantModuleManager()
            dormant_module_manager.initialize(godelos_integration, enhanced_websocket_manager or websocket_manager)
            logger.info("✅ Dormant module manager initialized — 8 cognitive subsystems activated")
        except Exception as e:
            logger.error(f"❌ Failed to initialize dormant module manager: {e}")
            dormant_module_manager = None

# REMOVED: continuous_cognitive_streaming() function
# This function was generating synthetic cognitive events every 4 seconds with hardcoded values.
# Real cognitive events should be generated by actual system state changes, not periodic broadcasting.

# Hot-reloader for ontology files (Issue #97)
_hot_reloader = None

@asynccontextmanager
async def lifespan(app: FastAPI):
    """Application lifespan manager."""
    global startup_time, self_modification_engine, _hot_reloader
    
    # Startup
    startup_time = time.time()
    logger.info("🚀 Starting GödelOS Unified Server...")
    
    # Initialize SelfModificationEngine
    try:
        from backend.core.self_modification_engine import SelfModificationEngine
        self_modification_engine = SelfModificationEngine()
        logger.info("✅ SelfModificationEngine initialized")
    except Exception as e:
        logger.exception("Failed to initialize SelfModificationEngine: %s", e)
        self_modification_engine = None
    
    # Initialize core services first
    await initialize_core_services()
    
    # Initialize optional services
    await initialize_optional_services()
    
    # Set up consciousness engine in endpoints after initialization
    if UNIFIED_CONSCIOUSNESS_AVAILABLE and unified_consciousness_engine and enhanced_websocket_manager:
        try:
            from backend.api.consciousness_endpoints import set_consciousness_engine
            set_consciousness_engine(unified_consciousness_engine, enhanced_websocket_manager)
            logger.info("✅ Consciousness engine connected to API endpoints")
        except Exception as e:
            logger.error(f"Failed to connect consciousness engine to endpoints: {e}")
    
    # Initialize consciousness emergence detector
    try:
        from backend.core.consciousness_emergence_detector import (
            ConsciousnessEmergenceDetector,
            UnifiedConsciousnessObservatory,
        )
        from backend.api.consciousness_endpoints import set_emergence_detector, set_observatory
        _detector = ConsciousnessEmergenceDetector(websocket_manager=websocket_manager)
        set_emergence_detector(_detector)
        _observatory = UnifiedConsciousnessObservatory(_detector)
        await _observatory.start()
        set_observatory(_observatory)
        logger.info("✅ Consciousness emergence detector and observatory initialized")
    except Exception as e:
        logger.error(f"Failed to initialize consciousness emergence detector: {e}")

    # Initialize autonomous goal generator and creative synthesis engine
    try:
        from backend.core.autonomous_goal_engine import AutonomousGoalGenerator, CreativeSynthesisEngine
        from backend.api.consciousness_endpoints import set_goal_engine
        _goal_generator = AutonomousGoalGenerator()
        _creative_engine = CreativeSynthesisEngine()
        set_goal_engine(_goal_generator, _creative_engine)
        # Seed with a baseline generation so goals exist immediately
        await _goal_generator.generate({})
        logger.info("✅ Autonomous goal generator and creative synthesis engine initialized")
    except Exception as e:
        logger.error(f"Failed to initialize autonomous goal engine: {e}")

    # Initialize ontology hot-reloader for knowledge graph persistence (Issue #97)
    try:
        import os as _os
        ontology_dir = _os.environ.get("GODELOS_ONTOLOGY_DIR", "")
        if ontology_dir:
            from godelOS.core_kr.knowledge_store.hot_reloader import OntologyHotReloader

            def _on_triple_add(subject, predicate, obj):
                logger.debug("Hot-reload: +triple (%s, %s, %s)", subject, predicate, obj)

            def _on_triple_remove(subject, predicate, obj):
                logger.debug("Hot-reload: -triple (%s, %s, %s)", subject, predicate, obj)

            _hot_reloader = OntologyHotReloader(
                watch_dir=ontology_dir,
                on_add=_on_triple_add,
                on_remove=_on_triple_remove,
            )
            _hot_reloader.start()
            logger.info("✅ Ontology hot-reloader watching %s", ontology_dir)
        else:
            logger.info("ℹ  Ontology hot-reloader inactive (set GODELOS_ONTOLOGY_DIR to enable)")
    except Exception as e:
        logger.error(f"Failed to initialize ontology hot-reloader: {e}")
    
    # Eagerly initialize the agentic daemon system so the singleton is created
    # with all available dependencies (especially consciousness_engine).
    try:
        from backend.core.agentic_daemon_system import get_agentic_daemon_system
        await get_agentic_daemon_system(
            cognitive_manager=cognitive_manager,
            knowledge_pipeline=knowledge_pipeline_service if KNOWLEDGE_SERVICES_AVAILABLE else None,
            websocket_manager=websocket_manager,
            consciousness_engine=unified_consciousness_engine,
        )
        logger.info("✅ Agentic daemon system initialized with consciousness engine")
    except Exception as e:
        logger.error(f"Failed to initialize agentic daemon system: {e}")
    
    # REMOVED: Synthetic cognitive streaming - replaced with real event-driven updates
    # cognitive_streaming_task = asyncio.create_task(continuous_cognitive_streaming())
    logger.info("✅ Synthetic cognitive streaming disabled - using event-driven updates only")

    # Start dormant-module ticker background task
    # Initialise to None before the conditional block so the shutdown section
    # can always safely check it regardless of whether startup succeeded.
    _dormant_ticker_task = None
    if dormant_module_manager is not None:
        async def _dormant_modules_ticker():
            """Tick all active dormant modules every 2 seconds (same cadence as consciousness loop)."""
            while True:
                try:
                    await dormant_module_manager.tick()
                except Exception as exc:
                    logger.debug("Dormant module ticker error: %s", exc)
                await asyncio.sleep(2.0)

        _dormant_ticker_task = asyncio.create_task(_dormant_modules_ticker())
        logger.info("🔄 Dormant module ticker started")

    logger.info("🎉 GödelOS Unified Server fully initialized!")
    
    yield
    
    # Shutdown
    logger.info("🛑 Shutting down GödelOS Unified Server...")

    if _dormant_ticker_task is not None:
        _dormant_ticker_task.cancel()
        try:
            await _dormant_ticker_task
        except asyncio.CancelledError:
            logger.debug("✅ Dormant module ticker stopped cleanly")

    # Stop the consciousness observatory if it was started
    try:
        from backend.api.consciousness_endpoints import get_observatory
        _obs = get_observatory()
        if _obs is not None:
            await _obs.stop()
    except Exception:
        pass

    # Stop the ontology hot-reloader if active
    try:
        if _hot_reloader is not None:
            _hot_reloader.stop()
    except Exception:
        pass


    # No synthetic streaming task to cancel
    logger.info("✅ Shutdown complete")

# Server start time for metrics
server_start_time = time.time()

# Create FastAPI app
app = FastAPI(
    title="GödelOS Unified Cognitive API",
    description="Consolidated cognitive architecture API with full functionality",
    version="2.0.0",
    lifespan=lifespan
)

# Configure CORS
app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],  # In production, replace with specific origins
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

# Include enhanced routers if available
# NOTE: Disabling external enhanced_cognitive_router as we have local implementations
if ENHANCED_APIS_AVAILABLE:
    # Skip enhanced_cognitive_router to avoid conflicts with local endpoints
    # if enhanced_cognitive_router:
    #     app.include_router(enhanced_cognitive_router, prefix="/api/enhanced-cognitive", tags=["Enhanced Cognitive API"])
    if transparency_router:
        app.include_router(transparency_router)

# Include vector database router
if VECTOR_DATABASE_AVAILABLE and vector_db_router:
    app.include_router(vector_db_router, tags=["Vector Database Management"])

# Include distributed vector database router
if DISTRIBUTED_VECTOR_AVAILABLE and distributed_vector_router:
    app.include_router(distributed_vector_router, prefix="/api/distributed-vector", tags=["Distributed Vector Search"])

# Include adaptive ingestion router
try:
    from backend.api.adaptive_ingestion_endpoints import router as adaptive_ingestion_router
    app.include_router(adaptive_ingestion_router, tags=["Adaptive Ingestion"])
    logger.info("Adaptive ingestion endpoints included")
except ImportError as e:
    logger.warning(f"Adaptive ingestion endpoints not available: {e}")
except Exception as e:
    logger.error(f"Failed to setup adaptive ingestion endpoints: {e}")

# Include agentic daemon management router
try:
    from backend.api.agentic_daemon_endpoints import router as agentic_daemon_router
    app.include_router(agentic_daemon_router, tags=["Agentic Daemon System"])
    AGENTIC_DAEMON_AVAILABLE = True
    logger.info("Agentic daemon management endpoints available")
except ImportError as e:
    logger.warning(f"Agentic daemon endpoints not available: {e}")
    AGENTIC_DAEMON_AVAILABLE = False
except Exception as e:
    logger.error(f"Failed to setup agentic daemon endpoints: {e}")
    AGENTIC_DAEMON_AVAILABLE = False

# Include enhanced knowledge management router
try:
    from backend.api.knowledge_management_endpoints import router as knowledge_management_router
    app.include_router(knowledge_management_router, tags=["Knowledge Management"])
    KNOWLEDGE_MANAGEMENT_AVAILABLE = True
    logger.info("Enhanced knowledge management endpoints available")
except ImportError as e:
    logger.warning(f"Knowledge management endpoints not available: {e}")
    KNOWLEDGE_MANAGEMENT_AVAILABLE = False
except Exception as e:
    logger.error(f"Failed to setup knowledge management endpoints: {e}")
    KNOWLEDGE_MANAGEMENT_AVAILABLE = False

# Include unified consciousness endpoints
try:
    from backend.api.consciousness_endpoints import router as consciousness_router, set_consciousness_engine
    app.include_router(consciousness_router, tags=["Unified Consciousness"])
    
    # Set consciousness engine reference after initialization
    if UNIFIED_CONSCIOUSNESS_AVAILABLE and unified_consciousness_engine and enhanced_websocket_manager:
        set_consciousness_engine(unified_consciousness_engine, enhanced_websocket_manager)
        logger.info("✅ Unified consciousness endpoints available with engine integration")
    else:
        logger.info("✅ Unified consciousness endpoints available (engine will be set later)")
    
    CONSCIOUSNESS_ENDPOINTS_AVAILABLE = True
except ImportError as e:
    logger.warning(f"Consciousness endpoints not available: {e}")
    CONSCIOUSNESS_ENDPOINTS_AVAILABLE = False
except Exception as e:
    logger.error(f"Failed to setup consciousness endpoints: {e}")
    CONSCIOUSNESS_ENDPOINTS_AVAILABLE = False

# Setup replay harness endpoints
try:
    from backend.api.replay_endpoints import setup_replay_endpoints
    setup_replay_endpoints(app, None)  # Will be updated with cognitive_manager once available
    logger.info("Replay harness endpoints initialized")
except ImportError as e:
    logger.warning(f"Replay endpoints not available: {e}")
except Exception as e:
    logger.error(f"Failed to setup replay endpoints: {e}")

# Include external API router (REST + WebSocket surface for external consumers)
try:
    from backend.api.external_api import router as external_api_router, configure as configure_external_api
    app.include_router(external_api_router, tags=["External API"])
    # Wire runtime dependencies into the external API module.
    configure_external_api(
        godelos_integration=godelos_integration,
        websocket_manager=websocket_manager,
        cognitive_state=cognitive_state,
        startup_time=server_start_time,
        tool_based_llm=tool_based_llm,
    )
    EXTERNAL_API_AVAILABLE = True
    logger.info("External API endpoints available at /api/v1/external/*")
except ImportError as e:
    logger.warning(f"External API endpoints not available: {e}")
    EXTERNAL_API_AVAILABLE = False
except Exception as e:
    logger.error(f"Failed to setup external API endpoints: {e}")
    EXTERNAL_API_AVAILABLE = False

# Consciousness bootstrap endpoint (manual trigger for 6-phase bootstrap)
@app.post("/api/consciousness/bootstrap")
async def api_consciousness_bootstrap(force: bool = Query(default=False)):
    """Trigger the consciousness bootstrap sequence and stream progress.

    Uses CognitiveManager.consciousness_engine if available.
    Returns skipped if already completed and force is False.
    """
    try:
        if not cognitive_manager or not hasattr(cognitive_manager, 'consciousness_engine') or cognitive_manager.consciousness_engine is None:
            raise HTTPException(status_code=503, detail="Consciousness engine not available")

        ce = cognitive_manager.consciousness_engine
        # Skip duplicate unless force
        if hasattr(ce, 'is_bootstrap_complete') and ce.is_bootstrap_complete() and not force:
            # Notify clients that bootstrap is already complete
            try:
                if enhanced_websocket_manager and hasattr(enhanced_websocket_manager, 'broadcast_consciousness_update'):
                    await enhanced_websocket_manager.broadcast_consciousness_update({
                        'type': 'bootstrap_progress',
                        'phase': 'Already Completed',
                        'awareness_level': getattr(ce.current_state, 'awareness_level', 1.0) or 1.0,
                        'timestamp': time.time(),
                        'message': 'Bootstrap already completed'
                    })
            except Exception:
                pass
            return {"status": "skipped", "message": "Bootstrap already completed"}

        # Announce initiation for immediate UI feedback
        try:
            if enhanced_websocket_manager and hasattr(enhanced_websocket_manager, 'broadcast_consciousness_update'):
                await enhanced_websocket_manager.broadcast_consciousness_update({
                    'type': 'bootstrap_progress',
                    'phase': 'Bootstrap Initiated',
                    'awareness_level': getattr(ce.current_state, 'awareness_level', 0.0) or 0.0,
                    'timestamp': time.time(),
                    'message': 'Starting 6-phase consciousness bootstrap'
                })
        except Exception:
            pass
        await ce.bootstrap_consciousness()
        return {"status": "started", "message": "Bootstrap sequence executed"}

    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error triggering consciousness bootstrap: {e}")
        raise HTTPException(status_code=500, detail=str(e))

# Root and health endpoints
@app.get("/")
async def root():
    """Root endpoint providing comprehensive API information."""
    return {
        "name": "GödelOS Unified Cognitive API",
        "version": "2.0.0",
        "status": "operational",
        "services": {
            "godelos_integration": GODELOS_AVAILABLE and godelos_integration is not None,
            "llm_integration": LLM_INTEGRATION_AVAILABLE and tool_based_llm is not None,
            "knowledge_services": KNOWLEDGE_SERVICES_AVAILABLE,
            "enhanced_apis": ENHANCED_APIS_AVAILABLE,
            "websocket_streaming": websocket_manager is not None
        },
        "endpoints": {
            "core": ["/", "/health", "/api/health"],
            "cognitive": ["/cognitive/state", "/api/cognitive/state"],
            "llm": ["/api/llm-chat/message", "/api/llm-tools/test", "/api/llm-tools/available"],
            "streaming": ["/ws/cognitive-stream"],
            "enhanced": ["/api/enhanced-cognitive/*", "/api/transparency/*"] if ENHANCED_APIS_AVAILABLE else []
        },
        "features": [
            "Unified server architecture",
            "Tool-based LLM integration",
            "Real-time cognitive streaming", 
            "Advanced knowledge processing",
            "Cognitive transparency",
            "WebSocket live updates"
        ]
    }

@app.get("/health")
async def health_check():
    """Comprehensive health check endpoint with subsystem probes."""
    # Base service status
    services = {
        "godelos": "active" if godelos_integration else "inactive",
        "llm_tools": "active" if tool_based_llm else "inactive",
        "websockets": f"{len(websocket_manager.active_connections) if websocket_manager and hasattr(websocket_manager, 'active_connections') else 0} connections"
    }

    # Subsystem probes (best-effort; never raise)
    probes = {}

    # Vector DB probe
    try:
        if VECTOR_DATABASE_AVAILABLE and get_vector_database is not None:
            vdb = get_vector_database()
            probes["vector_database"] = vdb.health_check() if hasattr(vdb, "health_check") else {"status": "unknown"}
        else:
            probes["vector_database"] = {"status": "unavailable"}
    except Exception as e:
        probes["vector_database"] = {"status": "error", "message": str(e)}

    # Knowledge pipeline probe (sync stats)
    try:
        if KNOWLEDGE_SERVICES_AVAILABLE and knowledge_pipeline_service is not None:
            probes["knowledge_pipeline"] = knowledge_pipeline_service.get_statistics()
        else:
            probes["knowledge_pipeline"] = {"status": "unavailable"}
    except Exception as e:
        probes["knowledge_pipeline"] = {"status": "error", "message": str(e)}

    # Knowledge ingestion probe (queue size, initialized)
    try:
        if KNOWLEDGE_SERVICES_AVAILABLE and knowledge_ingestion_service is not None:
            initialized = getattr(knowledge_ingestion_service, "processing_task", None) is not None
            queue_size = getattr(getattr(knowledge_ingestion_service, "import_queue", None), "qsize", lambda: 0)()
            probes["knowledge_ingestion"] = {"initialized": initialized, "queue_size": queue_size, "status": "healthy" if initialized else "initializing"}
        else:
            probes["knowledge_ingestion"] = {"status": "unavailable"}
    except Exception as e:
        probes["knowledge_ingestion"] = {"status": "error", "message": str(e)}

    # Cognitive manager probe
    try:
        if cognitive_manager is not None:
            active_sessions = len(getattr(cognitive_manager, "active_sessions", {}) or {})
            probes["cognitive_manager"] = {"initialized": True, "active_sessions": active_sessions, "status": "healthy"}
        else:
            probes["cognitive_manager"] = {"status": "unavailable"}
    except Exception as e:
        probes["cognitive_manager"] = {"status": "error", "message": str(e)}

    # Enhanced APIs / transparency
    try:
        probes["enhanced_apis"] = {"available": ENHANCED_APIS_AVAILABLE, "status": "healthy" if ENHANCED_APIS_AVAILABLE else "unavailable"}
    except Exception:
        probes["enhanced_apis"] = {"status": "unknown"}

    # Agentic daemon system probe
    try:
        probes["agentic_daemon_system"] = {"available": AGENTIC_DAEMON_AVAILABLE, "status": "healthy" if AGENTIC_DAEMON_AVAILABLE else "unavailable"}
    except Exception:
        probes["agentic_daemon_system"] = {"status": "unknown"}

    # Knowledge management system probe
    try:
        probes["knowledge_management_system"] = {"available": KNOWLEDGE_MANAGEMENT_AVAILABLE, "status": "healthy" if KNOWLEDGE_MANAGEMENT_AVAILABLE else "unavailable"}
    except Exception:
        probes["knowledge_management_system"] = {"status": "unknown"}

    now_iso = datetime.now().isoformat()
    # Stamp each probe with a timestamp to aid diagnostics
    for key in list(probes.keys()):
        try:
            if isinstance(probes[key], dict) and "timestamp" not in probes[key]:
                probes[key]["timestamp"] = time.time()
        except Exception:
            pass

    return {
        "status": "healthy",
        "timestamp": now_iso,
        "probe_timestamp": now_iso,
        "services": services,
        "probes": probes,
        "version": "2.0.0"
    }

@app.get("/metrics")
async def get_metrics():
    """Enhanced Prometheus-style metrics endpoint with comprehensive observability."""
    try:
        # Use enhanced metrics collector
        prometheus_output = metrics_collector.export_prometheus()
        
        return Response(
            content=prometheus_output,
            media_type="text/plain; version=0.0.4; charset=utf-8"
        )
        
    except Exception as e:
        logger.error(f"Error generating metrics: {e}")
        # Fallback to basic metrics
        return await get_basic_metrics()

async def get_basic_metrics():
    """Fallback basic metrics when enhanced metrics fail."""
    try:
        # Basic system metrics without psutil dependency
        import os
        from datetime import datetime
        
        # Process metrics
        process_start_time = time.time() - 3600  # Approximate
        
        # Cognitive manager metrics
        cognitive_metrics = {}
        if cognitive_manager:
            try:
                coordination_count = len(cognitive_manager.coordination_events)
                cognitive_metrics = {
                    "coordination_decisions_total": coordination_count,
                    "coordination_queue_size": coordination_count
                }
            except Exception:
                pass
        
        # Vector DB metrics
        vector_metrics = {}
        if VECTOR_DATABASE_AVAILABLE and get_vector_database:
            try:
                vdb = get_vector_database()
                if vdb:
                    # Get vector DB status
                    vector_status = getattr(vdb, '_last_probe_status', 'unknown')
                    vector_metrics = {
                        "vector_db_status": 1 if vector_status == 'healthy' else 0,
                        "vector_db_last_probe": getattr(vdb, '_last_probe_time', 0)
                    }
            except Exception:
                pass
        
        # WebSocket metrics
        websocket_metrics = {}
        if websocket_manager:
            try:
                active_connections = len(getattr(websocket_manager, 'active_connections', []))
                websocket_metrics = {
                    "websocket_connections_active": active_connections,
                    "websocket_messages_sent_total": getattr(websocket_manager, '_messages_sent', 0)
                }
            except Exception:
                pass
        
        metrics = {
            # Application metrics
            "godelos_version": "2.0.0",
            "godelos_start_time": server_start_time,
            "godelos_uptime_seconds": time.time() - server_start_time,
            
            **cognitive_metrics,
            **vector_metrics,
            **websocket_metrics
        }
        
        # Format as Prometheus-style text (basic implementation)
        prometheus_output = []
        for metric_name, value in metrics.items():
            if isinstance(value, (int, float)):
                prometheus_output.append(f"{metric_name} {value}")
            else:
                prometheus_output.append(f'# {metric_name} "{value}"')
        
        return Response(
            content="\n".join(prometheus_output) + "\n",
            media_type="text/plain"
        )
        
    except Exception as e:
        logger.error(f"Error generating metrics: {e}")
        return Response(
            content=f"# Error generating metrics: {e}\n",
            media_type="text/plain",
            status_code=500
        )

@app.get("/api/health")
async def api_health_check():
    """API health check endpoint with /api prefix."""
    return await health_check()

# Cognitive state endpoints
@app.get("/cognitive/state")
async def get_cognitive_state_endpoint():
    """Get current cognitive state."""
    if godelos_integration:
        try:
            return await godelos_integration.get_cognitive_state()
        except Exception as e:
            logger.error(f"Error getting cognitive state from GödelOS: {e}")
    
    # Return fallback state
    import random
    cognitive_state["processing_load"] = max(0, min(1, cognitive_state["processing_load"] + random.uniform(-0.1, 0.1)))
    return cognitive_state

@app.get("/api/cognitive/state") 
async def api_get_cognitive_state():
    """API cognitive state endpoint with /api prefix."""
    return await get_cognitive_state_endpoint()

@app.get("/api/cognitive-state") 
async def api_get_cognitive_state_alias():
    """API cognitive state endpoint with canonical data contract."""
    try:
        # Get data from GödelOS integration if available
        godelos_data = None
        if godelos_integration:
            try:
                godelos_data = await godelos_integration.get_cognitive_state()
            except Exception as e:
                logger.error(f"Error getting cognitive state from GödelOS: {e}")
        
        # Build canonical response with both camelCase and snake_case
        manifest_consciousness = get_manifest_consciousness_canonical()
        
        # If we have GödelOS data, merge it with manifest consciousness
        if godelos_data and "manifest_consciousness" in godelos_data:
            legacy_manifest = godelos_data["manifest_consciousness"]
            # Extract relevant data but keep canonical structure
            if "attention_focus" in legacy_manifest:
                focus = legacy_manifest["attention_focus"]
                if isinstance(focus, dict) and "primary" in focus:
                    manifest_consciousness["attention"]["focus"] = [focus["primary"]]
                    manifest_consciousness["attention"]["intensity"] = focus.get("intensity", 0.7)
            
            if "metacognitive_status" in godelos_data:
                meta = godelos_data["metacognitive_status"]
                if isinstance(meta, dict):
                    manifest_consciousness["metaReflection"]["depth"] = meta.get("self_awareness", 0.6)
                    manifest_consciousness["metaReflection"]["coherence"] = meta.get("confidence", 0.85)
        
        # Build canonical response
        canonical_response = {
            "version": "v1",
            "systemHealth": get_system_health_with_labels(),
            "manifestConsciousness": manifest_consciousness,
            "knowledgeStats": get_knowledge_stats(),
            # Legacy compatibility (snake_case mirror)
            "manifest_consciousness": manifest_consciousness,
        }
        
        return canonical_response
        
    except Exception as e:
        logger.error(f"Error building cognitive state response: {e}")
        # Return minimal fallback that satisfies the contract
        return {
            "version": "v1",
            "systemHealth": {
                "websocketConnection": 0.0,
                "pipeline": 0.0,
                "knowledgeStore": 0.0,
                "vectorIndex": 0.0,
                "_labels": {
                    "websocketConnection": "unknown",
                    "pipeline": "unknown", 
                    "knowledgeStore": "unknown",
                    "vectorIndex": "unknown"
                }
            },
            "manifestConsciousness": get_manifest_consciousness_canonical(),
            "knowledgeStats": get_knowledge_stats(),
            "manifest_consciousness": get_manifest_consciousness_canonical(),
        }

# Consciousness endpoints
@app.get("/api/v1/consciousness/state")
async def get_consciousness_state():
    """Get current consciousness state assessment."""
    try:
        if not cognitive_manager:
            raise _structured_http_error(503, code="cognitive_manager_unavailable", message="Consciousness engine not available", service="consciousness")
        
        consciousness_state = await cognitive_manager.assess_consciousness()
        return consciousness_state
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error getting consciousness state: {e}")
        raise _structured_http_error(500, code="consciousness_assessment_error", message=str(e), service="consciousness")

@app.post("/api/v1/consciousness/assess")
async def assess_consciousness():
    """Trigger a comprehensive consciousness assessment."""
    try:
        if not cognitive_manager:
            raise _structured_http_error(503, code="cognitive_manager_unavailable", message="Consciousness engine not available", service="consciousness")
        
        assessment = await cognitive_manager.assess_consciousness()
        return {
            "assessment": assessment,
            "timestamp": datetime.now().isoformat(),
            "status": "completed"
        }
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error assessing consciousness: {e}")
        raise _structured_http_error(500, code="consciousness_assessment_error", message=str(e), service="consciousness")

@app.get("/api/v1/consciousness/summary")
async def get_consciousness_summary():
    """Get a summary of consciousness capabilities and current state."""
    try:
        if not cognitive_manager:
            raise _structured_http_error(503, code="cognitive_manager_unavailable", message="Consciousness engine not available", service="consciousness")
        
        summary = await cognitive_manager.get_consciousness_summary()
        return summary
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error getting consciousness summary: {e}")
        raise _structured_http_error(500, code="consciousness_summary_error", message=str(e), service="consciousness")

@app.post("/api/v1/consciousness/goals/generate")
async def generate_autonomous_goals():
    """Generate autonomous goals based on current consciousness state."""
    try:
        if not cognitive_manager:
            raise _structured_http_error(503, code="cognitive_manager_unavailable", message="Consciousness engine not available", service="consciousness")
        
        goals = await cognitive_manager.initiate_autonomous_goals()
        return {
            "goals": goals,
            "timestamp": datetime.now().isoformat(),
            "status": "generated"
        }
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error generating autonomous goals: {e}")
        raise _structured_http_error(500, code="goal_generation_error", message=str(e), service="consciousness")

@app.get("/api/v1/consciousness/trajectory")
async def get_consciousness_trajectory():
    """Get consciousness trajectory and behavioral patterns."""
    try:
        if not cognitive_manager:
            raise _structured_http_error(503, code="cognitive_manager_unavailable", message="Consciousness engine not available", service="consciousness")
        
        # Get current state as baseline for trajectory
        current_state = await cognitive_manager.assess_consciousness()
        
        trajectory = {
            "current_state": current_state,
            "behavioral_patterns": {
                "autonomy_level": current_state.get("autonomy_level", 0.0),
                "self_awareness": current_state.get("self_awareness_level", 0.0),
                "intentionality": current_state.get("intentionality_strength", 0.0),
                "phenomenal_awareness": current_state.get("phenomenal_awareness", 0.0)
            },
            "trajectory_analysis": {
                "trend": "stable",
                "confidence": 0.8,
                "notable_changes": []
            },
            "timestamp": datetime.now().isoformat()
        }
        
        return trajectory
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error getting consciousness trajectory: {e}")
        raise _structured_http_error(500, code="consciousness_trajectory_error", message=str(e), service="consciousness")

# Transparency API endpoints
@app.get("/api/v1/transparency/metrics")
async def get_transparency_metrics():
    """Get current cognitive transparency metrics"""
    try:
        metrics = await transparency_engine.get_transparency_metrics()
        return JSONResponse(content=metrics)
    except Exception as e:
        logger.error(f"Error getting transparency metrics: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/api/v1/transparency/activity") 
async def get_cognitive_activity():
    """Get summary of recent cognitive activity"""
    try:
        activity = await transparency_engine.get_cognitive_activity_summary()
        return JSONResponse(content=activity)
    except Exception as e:
        logger.error(f"Error getting cognitive activity: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/api/v1/transparency/events")
async def get_recent_events(limit: int = Query(default=20, le=100)):
    """Get recent cognitive events"""
    try:
        events = transparency_engine.event_buffer[-limit:] if len(transparency_engine.event_buffer) >= limit else transparency_engine.event_buffer
        return JSONResponse(content={
            "events": [event.to_dict() for event in events],
            "total_events": len(transparency_engine.event_buffer),
            "returned_count": len(events)
        })
    except Exception as e:
        logger.error(f"Error getting recent events: {e}")
        raise HTTPException(status_code=500, detail=str(e))

# Meta-cognitive API endpoints
@app.post("/api/v1/metacognitive/monitor")
async def initiate_metacognitive_monitoring(context: Dict[str, Any] = None):
    """Initiate comprehensive meta-cognitive monitoring"""
    try:
        if not cognitive_manager:
            raise HTTPException(status_code=503, detail="Cognitive manager not available")
        
        result = await cognitive_manager.initiate_meta_cognitive_monitoring(context or {})
        return JSONResponse(content=result)
    except Exception as e:
        logger.error(f"Error initiating meta-cognitive monitoring: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/api/v1/metacognitive/analyze")
async def perform_metacognitive_analysis(request: QueryRequest):
    """Perform deep meta-cognitive analysis of a query"""
    try:
        if not cognitive_manager:
            raise HTTPException(status_code=503, detail="Cognitive manager not available")
        
        analysis = await cognitive_manager.perform_meta_cognitive_analysis(
            request.query, 
            request.context or {}
        )
        return JSONResponse(content=analysis)
    except Exception as e:
        logger.error(f"Error in meta-cognitive analysis: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/api/v1/metacognitive/self-awareness")
async def assess_self_awareness():
    """Assess current self-awareness level"""
    try:
        if not cognitive_manager:
            raise HTTPException(status_code=503, detail="Cognitive manager not available")
        
        assessment = await cognitive_manager.assess_self_awareness()
        return JSONResponse(content=assessment)
    except Exception as e:
        logger.error(f"Error in self-awareness assessment: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/api/v1/metacognitive/summary")
async def get_metacognitive_summary():
    """Get comprehensive meta-cognitive summary"""
    try:
        if not cognitive_manager:
            raise HTTPException(status_code=503, detail="Cognitive manager not available")
        
        summary = await cognitive_manager.get_meta_cognitive_summary()
        return JSONResponse(content=summary)
    except Exception as e:
        logger.error(f"Error getting meta-cognitive summary: {e}")
        raise HTTPException(status_code=500, detail=str(e))

# Autonomous Learning API endpoints
@app.post("/api/v1/learning/analyze-gaps")
async def analyze_knowledge_gaps(context: Dict[str, Any] = None):
    """Analyze and identify knowledge gaps for learning"""
    try:
        if not cognitive_manager:
            raise HTTPException(status_code=503, detail="Cognitive manager not available")
        
        result = await cognitive_manager.analyze_knowledge_gaps(context)
        return JSONResponse(content=result)
    except Exception as e:
        logger.error(f"Error analyzing knowledge gaps: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/api/v1/learning/generate-goals")
async def generate_autonomous_goals(
    focus_domains: List[str] = Query(default=None),
    urgency: str = Query(default="medium")
):
    """Generate autonomous learning goals"""
    try:
        if not cognitive_manager:
            raise HTTPException(status_code=503, detail="Cognitive manager not available")
        
        result = await cognitive_manager.generate_autonomous_learning_goals(
            focus_domains=focus_domains,
            urgency=urgency
        )
        return JSONResponse(content=result)
    except Exception as e:
        logger.error(f"Error generating autonomous goals: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/api/v1/learning/create-plan")
async def create_learning_plan(goal_ids: List[str] = Query(default=None)):
    """Create comprehensive learning plan"""
    try:
        if not cognitive_manager:
            raise HTTPException(status_code=503, detail="Cognitive manager not available")
        
        result = await cognitive_manager.create_learning_plan(goal_ids)
        return JSONResponse(content=result)
    except Exception as e:
        logger.error(f"Error creating learning plan: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/api/v1/learning/assess-skills")
async def assess_learning_skills(domains: List[str] = Query(default=None)):
    """Assess current skill levels across learning domains"""
    try:
        if not cognitive_manager:
            raise HTTPException(status_code=503, detail="Cognitive manager not available")
        
        result = await cognitive_manager.assess_learning_skills(domains)
        return JSONResponse(content=result)
    except Exception as e:
        logger.error(f"Error assessing learning skills: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/api/v1/learning/track-progress/{goal_id}")
async def track_learning_progress(goal_id: str, progress_data: Dict[str, Any]):
    """Track progress on a learning goal"""
    try:
        if not cognitive_manager:
            raise HTTPException(status_code=503, detail="Cognitive manager not available")
        
        result = await cognitive_manager.track_learning_progress(goal_id, progress_data)
        return JSONResponse(content=result)
    except Exception as e:
        logger.error(f"Error tracking learning progress: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/api/v1/learning/insights")
async def get_learning_insights():
    """Get insights about learning patterns and effectiveness"""
    try:
        if not cognitive_manager:
            raise HTTPException(status_code=503, detail="Cognitive manager not available")
        
        result = await cognitive_manager.get_learning_insights()
        return JSONResponse(content=result)
    except Exception as e:
        logger.error(f"Error getting learning insights: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/api/v1/learning/summary")
async def get_learning_summary():
    """Get comprehensive autonomous learning system summary"""
    try:
        if not cognitive_manager:
            raise HTTPException(status_code=503, detail="Cognitive manager not available")
        
        result = await cognitive_manager.get_autonomous_learning_summary()
        return JSONResponse(content=result)
    except Exception as e:
        logger.error(f"Error getting learning summary: {e}")
        raise HTTPException(status_code=500, detail=str(e))

# =====================================================================
# KNOWLEDGE GRAPH EVOLUTION ENDPOINTS
# =====================================================================

@app.post("/api/v1/knowledge-graph/evolve")
async def evolve_knowledge_graph(evolution_data: Dict[str, Any]):
    """Trigger knowledge graph evolution with automatic phenomenal experience integration"""
    try:
        if not cognitive_manager:
            raise HTTPException(status_code=503, detail="Cognitive manager not available")
        
        trigger = evolution_data.get("trigger")
        context = evolution_data.get("context", {})
        
        if not trigger:
            raise HTTPException(status_code=400, detail="Trigger is required")
        
        # Use integrated method that automatically triggers corresponding experiences
        result = await cognitive_manager.evolve_knowledge_graph_with_experience_trigger(
            trigger=trigger,
            context=context
        )
        return JSONResponse(content=result)
    except Exception as e:
        logger.error(f"Error evolving knowledge graph: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/api/v1/knowledge-graph/concepts")
async def add_knowledge_concept(concept_data: Dict[str, Any]):
    """Add a new concept to the knowledge graph"""
    try:
        if not cognitive_manager:
            raise HTTPException(status_code=503, detail="Cognitive manager not available")
        
        auto_connect = concept_data.get("auto_connect", True)
        result = await cognitive_manager.add_knowledge_concept(
            concept_data=concept_data,
            auto_connect=auto_connect
        )
        return JSONResponse(content=result)
    except Exception as e:
        logger.error(f"Error adding knowledge concept: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/api/v1/knowledge-graph/relationships")
async def create_knowledge_relationship(relationship_data: Dict[str, Any]):
    """Create a relationship between knowledge concepts"""
    try:
        if not cognitive_manager:
            raise HTTPException(status_code=503, detail="Cognitive manager not available")
        
        source_concept = relationship_data.get("source_id")
        target_concept = relationship_data.get("target_id") 
        relationship_type = relationship_data.get("relationship_type")
        strength = relationship_data.get("strength", 0.5)
        evidence = relationship_data.get("evidence", [])
        
        if not source_concept or not target_concept or not relationship_type:
            raise HTTPException(status_code=400, detail="source_id, target_id, and relationship_type are required")
        
        result = await cognitive_manager.create_knowledge_relationship(
            source_concept=source_concept,
            target_concept=target_concept,
            relationship_type=relationship_type,
            strength=strength,
            evidence=evidence
        )
        return JSONResponse(content=result)
    except Exception as e:
        logger.error(f"Error creating knowledge relationship: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/api/v1/knowledge-graph/patterns/detect")
async def detect_emergent_patterns():
    """Detect emergent patterns in the knowledge graph"""
    try:
        if not cognitive_manager:
            raise HTTPException(status_code=503, detail="Cognitive manager not available")
        
        result = await cognitive_manager.detect_emergent_patterns()
        return JSONResponse(content=result)
    except Exception as e:
        logger.error(f"Error detecting emergent patterns: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/api/v1/knowledge-graph/concepts/{concept_id}/neighborhood")
async def get_concept_neighborhood(
    concept_id: str,
    depth: int = Query(default=2, description="Depth of neighborhood analysis")
):
    """Get the neighborhood of concepts around a given concept"""
    try:
        if not cognitive_manager:
            raise _structured_http_error(503, code="cognitive_manager_unavailable", message="Cognitive manager not available", service="knowledge_graph")
        
        result = await cognitive_manager.get_concept_neighborhood(
            concept_id=concept_id,
            depth=depth
        )
        return JSONResponse(content=result)
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error getting concept neighborhood: {e}")
        raise _structured_http_error(500, code="kg_neighborhood_error", message=str(e), service="knowledge_graph")

@app.get("/api/v1/knowledge-graph/summary")
async def get_knowledge_graph_summary():
    """Get comprehensive summary of knowledge graph evolution"""
    try:
        if not cognitive_manager:
            raise _structured_http_error(503, code="cognitive_manager_unavailable", message="Cognitive manager not available", service="knowledge_graph")
        
        result = await cognitive_manager.get_knowledge_graph_summary()
        return JSONResponse(content=result)
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error getting knowledge graph summary: {e}")
        raise _structured_http_error(500, code="kg_summary_error", message=str(e), service="knowledge_graph")

# LLM KNOWLEDGE MINING ENDPOINTS

@app.post("/api/v1/knowledge-mining/bootstrap")
async def bootstrap_knowledge_from_llm():
    """Bootstrap the knowledge graph with LLM-generated system knowledge"""
    try:
        from backend.llm_knowledge_miner import get_llm_knowledge_miner
        
        logger.info("Starting LLM knowledge mining bootstrap")
        
        # Get the knowledge miner
        miner = await get_llm_knowledge_miner(
            llm_driver=llm_driver if 'llm_driver' in globals() else None,
            knowledge_graph=cognitive_manager.knowledge_graph_evolution if cognitive_manager else None
        )
        
        # Bootstrap system knowledge
        result = await miner.bootstrap_system_knowledge()
        
        logger.info(f"Bootstrap completed: {result}")
        return JSONResponse(content={
            "status": "success",
            "message": "Knowledge graph bootstrapped successfully",
            "result": result
        })
        
    except Exception as e:
        logger.error(f"Error bootstrapping knowledge from LLM: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/api/v1/knowledge-mining/mine-domain")
async def mine_domain_knowledge(request: Dict[str, Any]):
    """Mine knowledge for a specific domain from the LLM"""
    try:
        from backend.llm_knowledge_miner import get_llm_knowledge_miner
        
        domain = request.get("domain")
        if not domain:
            raise HTTPException(status_code=400, detail="Domain name is required")
        
        depth = request.get("depth", 2)
        if depth not in [1, 2, 3]:
            raise HTTPException(status_code=400, detail="Depth must be 1 (basic), 2 (intermediate), or 3 (comprehensive)")
        
        logger.info(f"Mining domain knowledge: {domain} at depth {depth}")
        
        # Get the knowledge miner
        miner = await get_llm_knowledge_miner(
            llm_driver=llm_driver if 'llm_driver' in globals() else None,
            knowledge_graph=cognitive_manager.knowledge_graph_evolution if cognitive_manager else None
        )
        
        # Mine the domain
        result = await miner.mine_domain_knowledge(domain, depth)
        
        logger.info(f"Domain mining completed: {result}")
        return JSONResponse(content={
            "status": "success",
            "message": f"Successfully mined knowledge for domain: {domain}",
            "result": result
        })
        
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error mining domain knowledge: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/api/v1/knowledge-mining/mine-multiple-domains")
async def mine_multiple_domains(request: Dict[str, Any]):
    """Mine knowledge for multiple interconnected domains"""
    try:
        from backend.llm_knowledge_miner import get_llm_knowledge_miner
        
        domains = request.get("domains", [])
        if not domains or not isinstance(domains, list):
            raise HTTPException(status_code=400, detail="Domains list is required")
        
        if len(domains) > 10:
            raise HTTPException(status_code=400, detail="Maximum 10 domains allowed per request")
        
        logger.info(f"Mining multiple interconnected domains: {domains}")
        
        # Get the knowledge miner
        miner = await get_llm_knowledge_miner(
            llm_driver=llm_driver if 'llm_driver' in globals() else None,
            knowledge_graph=cognitive_manager.knowledge_graph_evolution if cognitive_manager else None
        )
        
        # Mine interconnected domains
        result = await miner.mine_interconnected_domains(domains)
        
        logger.info(f"Multi-domain mining completed: {result}")
        return JSONResponse(content={
            "status": "success",
            "message": f"Successfully mined {len(domains)} interconnected domains",
            "result": result
        })
        
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error mining multiple domains: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/api/v1/knowledge-mining/status")
async def get_knowledge_mining_status():
    """Get the status of the knowledge mining system"""
    try:
        from backend.llm_knowledge_miner import _llm_knowledge_miner
        
        if _llm_knowledge_miner is None:
            return JSONResponse(content={
                "initialized": False,
                "message": "Knowledge miner not yet initialized"
            })
        
        return JSONResponse(content={
            "initialized": True,
            "concepts_mined": len(_llm_knowledge_miner.mined_concepts),
            "relationships_mined": len(_llm_knowledge_miner.mined_relationships),
            "llm_driver_available": _llm_knowledge_miner.llm_driver is not None,
            "knowledge_graph_connected": _llm_knowledge_miner.knowledge_graph is not None
        })
        
    except Exception as e:
        logger.error(f"Error getting knowledge mining status: {e}")
        raise HTTPException(status_code=500, detail=str(e))

# PHENOMENAL EXPERIENCE ENDPOINTS

@app.post("/api/v1/phenomenal/generate-experience")
async def generate_phenomenal_experience(experience_data: Dict[str, Any]):
    """Generate a phenomenal experience with automatic knowledge graph evolution integration"""
    try:
        if not cognitive_manager:
            raise _structured_http_error(503, code="cognitive_manager_unavailable", message="Cognitive manager not available", service="phenomenal")
        
        experience_type = experience_data.get("experience_type", "cognitive")
        trigger_context = experience_data.get("trigger_context", experience_data.get("context", ""))
        desired_intensity = experience_data.get("desired_intensity", experience_data.get("intensity", 0.7))
        context = experience_data.get("context", {})
        
        # Use integrated method that automatically triggers corresponding KG evolution
        result = await cognitive_manager.generate_experience_with_kg_evolution(
            experience_type=experience_type,
            trigger_context=trigger_context,
            desired_intensity=desired_intensity,
            context=context
        )
        
        if result.get("error"):
            raise _structured_http_error(500, code="phenomenal_generation_error", message=str(result["error"]), service="phenomenal")
        
        return JSONResponse(content={
            "status": "success",
            "experience": result["experience"],
            "triggered_kg_evolutions": result.get("triggered_kg_evolutions", []),
            "integration_status": result.get("integration_status"),
            "bidirectional": result.get("bidirectional", False)
        })
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error generating phenomenal experience: {e}")
        raise _structured_http_error(500, code="phenomenal_generation_error", message=str(e), service="phenomenal")

@app.get("/api/v1/phenomenal/conscious-state")
async def get_conscious_state():
    """Get the current conscious state"""
    try:
        from backend.core.phenomenal_experience import phenomenal_experience_generator
        
        conscious_state = phenomenal_experience_generator.get_current_conscious_state()
        
        if not conscious_state:
            return JSONResponse(content={
                "status": "no_active_state",
                "message": "No current conscious state available"
            })
        
        return JSONResponse(content={
            "status": "success",
            "conscious_state": {
                "id": conscious_state.id,
                "active_experiences": [
                    {
                        "id": exp.id,
                        "type": exp.experience_type.value,
                        "narrative": exp.narrative_description,
                        "vividness": exp.vividness,
                        "attention_focus": exp.attention_focus
                    } for exp in conscious_state.active_experiences
                ],
                "background_tone": conscious_state.background_tone,
                "attention_distribution": conscious_state.attention_distribution,
                "self_awareness_level": conscious_state.self_awareness_level,
                "phenomenal_unity": conscious_state.phenomenal_unity,
                "narrative_self": conscious_state.narrative_self,
                "timestamp": conscious_state.timestamp
            }
        })
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error getting conscious state: {e}")
        raise _structured_http_error(500, code="phenomenal_state_error", message=str(e), service="phenomenal")
    
@app.get("/api/v1/cognitive/coordination/recent")
async def get_recent_coordination_decisions(
    limit: int = Query(default=20, le=100),
    session_id: Optional[str] = Query(default=None),
    min_confidence: Optional[float] = Query(default=None, ge=0.0, le=1.0),
    max_confidence: Optional[float] = Query(default=None, ge=0.0, le=1.0),
    augmentation_only: Optional[bool] = Query(default=None),
    since_timestamp: Optional[float] = Query(default=None)
):
    """Surface recent coordination decisions for observability (no PII) with filtering."""
    try:
        if not cognitive_manager:
            raise _structured_http_error(503, code="cognitive_manager_unavailable", message="Cognitive manager not available", service="coordination")
        
        # Get all decisions and apply filters
        all_decisions = cognitive_manager.get_recent_coordination_decisions(limit=1000)  # Get more to filter
        filtered_decisions = []
        
        for decision in all_decisions:
            # Apply filters
            if session_id and decision.get("session_id") != session_id:
                continue
            if min_confidence is not None and decision.get("confidence", 0.0) < min_confidence:
                continue
            if max_confidence is not None and decision.get("confidence", 1.0) > max_confidence:
                continue
            if augmentation_only is not None and decision.get("augmentation", False) != augmentation_only:
                continue
            if since_timestamp is not None and decision.get("timestamp", 0.0) < since_timestamp:
                continue
            
            filtered_decisions.append(decision)
        
        # Apply limit to filtered results
        final_decisions = filtered_decisions[-limit:] if limit > 0 else filtered_decisions
        
        return JSONResponse(content={
            "count": len(final_decisions),
            "total_before_limit": len(filtered_decisions),
            "limit": limit,
            "filters": {
                "session_id": session_id,
                "min_confidence": min_confidence,
                "max_confidence": max_confidence,
                "augmentation_only": augmentation_only,
                "since_timestamp": since_timestamp
            },
            "decisions": final_decisions
        })
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error getting coordination decisions: {e}")
        raise _structured_http_error(500, code="coordination_telemetry_error", message=str(e), service="coordination")

@app.get("/api/v1/phenomenal/experience-history")
async def get_experience_history(limit: Optional[int] = 10):
    """Get phenomenal experience history"""
    try:
        from backend.core.phenomenal_experience import phenomenal_experience_generator
        
        experiences = phenomenal_experience_generator.get_experience_history(limit=limit)
        
        return JSONResponse(content={
            "status": "success",
            "experiences": [
                {
                    "id": exp.id,
                    "type": exp.experience_type.value,
                    "narrative": exp.narrative_description,
                    "vividness": exp.vividness,
                    "coherence": exp.coherence,
                    "attention_focus": exp.attention_focus,
                    "temporal_extent": exp.temporal_extent,
                    "triggers": exp.causal_triggers,
                    "concepts": exp.associated_concepts,
                    "background_context": exp.background_context,
                    "metadata": exp.metadata
                } for exp in experiences
            ],
            "total_count": len(experiences)
        })
    except Exception as e:
        logger.error(f"Error getting experience history: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/api/v1/phenomenal/experience-summary")
async def get_experience_summary():
    """Get summary statistics about phenomenal experiences"""
    try:
        from backend.core.phenomenal_experience import phenomenal_experience_generator
        
        summary = phenomenal_experience_generator.get_experience_summary()
        
        return JSONResponse(content={
            "status": "success",
            "summary": summary
        })
    except Exception as e:
        logger.error(f"Error getting experience summary: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.post("/api/v1/phenomenal/trigger-experience")
async def trigger_specific_experience(trigger_data: Dict[str, Any]):
    """Trigger a specific type of phenomenal experience with detailed context"""
    try:
        if not cognitive_manager:
            raise HTTPException(status_code=503, detail="Cognitive manager not available")
        
        from backend.core.phenomenal_experience import phenomenal_experience_generator, ExperienceType
        
        experience_type_str = trigger_data.get("type", "cognitive")
        context = trigger_data.get("context", {})
        intensity = trigger_data.get("intensity", 0.7)
        
        # Enhanced context processing
        enhanced_context = {
            **context,
            "user_request": True,
            "triggered_at": time.time(),
            "request_id": str(uuid.uuid4())
        }
        
        # Convert string to enum
        try:
            experience_type = ExperienceType(experience_type_str.lower())
        except ValueError:
            available_types = [e.value for e in ExperienceType]
            raise HTTPException(
                status_code=400, 
                detail=f"Invalid experience type. Available types: {available_types}"
            )
        
        experience = await phenomenal_experience_generator.generate_experience(
            trigger_context=enhanced_context,
            experience_type=experience_type,
            desired_intensity=intensity
        )
        
        return JSONResponse(content={
            "status": "success",
            "message": f"Generated {experience_type.value} experience",
            "experience": {
                "id": experience.id,
                "type": experience.experience_type.value,
                "narrative": experience.narrative_description,
                "vividness": experience.vividness,
                "coherence": experience.coherence,
                "attention_focus": experience.attention_focus,
                "qualia_patterns": [
                    {
                        "modality": q.modality.value,
                        "intensity": q.intensity,
                        "valence": q.valence,
                        "complexity": q.complexity,
                        "duration": q.duration
                    } for q in experience.qualia_patterns
                ],
                "temporal_extent": experience.temporal_extent,
                "triggers": experience.causal_triggers
            }
        })
    except Exception as e:
        logger.error(f"Error triggering phenomenal experience: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/api/v1/phenomenal/available-types")
async def get_available_experience_types():
    """Get available phenomenal experience types"""
    try:
        from backend.core.phenomenal_experience import ExperienceType
        
        types = [
            {
                "type": exp_type.value,
                "description": {
                    "cognitive": "General thinking and reasoning experiences",
                    "emotional": "Affective and feeling-based experiences",
                    "sensory": "Sensory-like qualitative experiences",
                    "attention": "Focused attention and concentration experiences",
                    "memory": "Memory retrieval and temporal experiences",
                    "metacognitive": "Self-awareness and reflection experiences",
                    "imaginative": "Creative and imaginative experiences",
                    "social": "Interpersonal and communication experiences",
                    "temporal": "Time perception and temporal awareness",
                    "spatial": "Spatial reasoning and dimensional awareness"
                }.get(exp_type.value, "Conscious experience type")
            } for exp_type in ExperienceType
        ]
        
        return JSONResponse(content={
            "status": "success",
            "available_types": types,
            "total_types": len(types)
        })
    except Exception as e:
        logger.error(f"Error getting available experience types: {e}")
        raise HTTPException(status_code=500, detail=str(e))


# Cognitive Architecture Integration Endpoints

@app.post("/api/v1/cognitive/loop")
async def execute_cognitive_loop(loop_data: Dict[str, Any]):
    """Execute a full bidirectional cognitive loop with KG-PE integration"""
    correlation_id = correlation_tracker.generate_correlation_id()
    
    with correlation_tracker.request_context(correlation_id):
        with operation_timer("cognitive_loop"):
            try:
                logger.info("Starting cognitive loop execution", extra={
                    "operation": "cognitive_loop",
                    "trigger_type": loop_data.get("trigger_type", "knowledge"),
                    "loop_depth": loop_data.get("loop_depth", 3)
                })
                
                if not cognitive_manager:
                    logger.error("Cognitive manager not available")
                    raise HTTPException(status_code=503, detail="Cognitive manager not available")
                
                initial_trigger = loop_data.get("initial_trigger", "new_information")
                trigger_type = loop_data.get("trigger_type", "knowledge")  # "knowledge" or "experience"
                loop_depth = min(loop_data.get("loop_depth", 3), 10)  # Max 10 steps for safety
                context = loop_data.get("context", {})
                
                result = await cognitive_manager.process_cognitive_loop(
                    initial_trigger=initial_trigger,
                    trigger_type=trigger_type,
                    loop_depth=loop_depth,
                    context=context
                )
                
                logger.info("Cognitive loop completed successfully", extra={
                    "operation": "cognitive_loop",
                    "result_steps": len(result.get("steps", [])) if isinstance(result, dict) else 0
                })
                
                return JSONResponse(content={
                    "status": "success",
                    "cognitive_loop": result
                })
                
            except Exception as e:
                logger.error(f"Error executing cognitive loop: {e}", extra={
                    "operation": "cognitive_loop",
                    "error_type": type(e).__name__
                })
                raise HTTPException(status_code=500, detail=str(e))

# Knowledge endpoints
@app.get("/api/knowledge/concepts")
async def get_knowledge_concepts():
    """Get available knowledge concepts."""
    try:
        concepts = [
            {
                "id": "reasoning",
                "name": "Logical Reasoning",
                "description": "Core reasoning capabilities and inference patterns",
                "active": True
            },
            {
                "id": "memory",
                "name": "Memory Management",
                "description": "Working memory and long-term knowledge storage",
                "active": True
            },
            {
                "id": "learning",
                "name": "Adaptive Learning",
                "description": "Continuous learning and knowledge integration",
                "active": True
            },
            {
                "id": "metacognition",
                "name": "Meta-Cognitive Awareness",
                "description": "Self-awareness of cognitive processes",
                "active": True
            }
        ]
        return {
            "concepts": concepts,
            "total_count": len(concepts),
            "timestamp": datetime.now().isoformat()
        }
    except Exception as e:
        logger.error(f"Error retrieving knowledge concepts: {e}")
        raise HTTPException(status_code=500, detail=f"Knowledge system error: {str(e)}")

@app.get("/api/knowledge/graph")
async def get_knowledge_graph():
    """Get the UNIFIED knowledge graph structure - single source of truth."""
    try:
        # Import here to avoid circular dependency
        from backend.cognitive_transparency_integration import cognitive_transparency_api
        
        # UNIFIED SYSTEM: Only one knowledge graph source
        if cognitive_transparency_api and cognitive_transparency_api.knowledge_graph:
            try:
                # Get dynamic graph data from the UNIFIED transparency system
                graph_data = await cognitive_transparency_api.knowledge_graph.export_graph()
                
                # Return unified format
                return {
                    "nodes": graph_data.get("nodes", []),
                    "edges": graph_data.get("edges", []),
                    "metadata": {
                        "node_count": len(graph_data.get("nodes", [])),
                        "edge_count": len(graph_data.get("edges", [])),
                        "last_updated": datetime.now().isoformat(),
                        "data_source": "unified_dynamic_transparency_system"
                    }
                }
            except Exception as e:
                logger.warning(f"Failed to get unified dynamic knowledge graph: {e}")
                # Re-raise the error instead of falling back to static data
                raise HTTPException(status_code=500, detail=f"Knowledge graph error: {str(e)}")
        else:
            # System not ready - return empty graph, NO STATIC FALLBACK
            logger.warning("Cognitive transparency system not initialized")
            return {
                "nodes": [],
                "edges": [],
                "metadata": {
                    "node_count": 0,
                    "edge_count": 0,
                    "last_updated": datetime.now().isoformat(),
                    "data_source": "system_not_ready",
                    "error": "Cognitive transparency system not initialized"
                }
            }
        
    except Exception as e:
        logger.error(f"Error retrieving unified knowledge graph: {e}")
        raise HTTPException(status_code=500, detail=f"Knowledge graph error: {str(e)}")

@app.post("/api/knowledge/reanalyze")
async def reanalyze_all_documents():
    """Re-analyze all stored documents and rebuild the unified knowledge graph."""
    try:
        # Import here to avoid circular dependency
        from backend.cognitive_transparency_integration import cognitive_transparency_api
        from backend.knowledge_ingestion import knowledge_ingestion_service
        import glob
        import json
        
        if not cognitive_transparency_api or not cognitive_transparency_api.knowledge_graph:
            raise HTTPException(status_code=503, detail="Cognitive transparency system not ready")
        
        if not knowledge_ingestion_service:
            raise HTTPException(status_code=503, detail="Knowledge ingestion service not available")
        
        # Get all stored documents
        storage_path = knowledge_ingestion_service.storage_path
        if not storage_path or not storage_path.exists():
            return {"message": "No documents found to reanalyze", "processed": 0}
        
        # Find all JSON files
        json_files = glob.glob(str(storage_path / "*.json"))
        document_files = [f for f in json_files if not os.path.basename(f).startswith("temp_")]
        
        logger.info(f"🔄 Re-analyzing {len(document_files)} documents...")
        
        processed_count = 0
        failed_count = 0
        
        for file_path in document_files:
            try:
                # Load document data
                with open(file_path, 'r') as f:
                    doc_data = json.load(f)
                
                # Extract concepts for knowledge graph
                concepts = []
                
                # Add title
                if doc_data.get('title'):
                    concepts.append(doc_data['title'])
                
                # Add categories
                if doc_data.get('categories'):
                    concepts.extend(doc_data['categories'])
                
                # Add keywords from metadata
                if doc_data.get('metadata', {}).get('keywords'):
                    keywords = doc_data['metadata']['keywords']
                    if isinstance(keywords, list):
                        concepts.extend(keywords[:5])
                
                # Add concepts to unified knowledge graph
                for concept in concepts:
                    if concept and isinstance(concept, str) and len(concept.strip()) > 0:
                        await cognitive_transparency_api.knowledge_graph.add_node(
                            concept=concept.strip(),
                            node_type="knowledge_item",
                            properties={
                                "source_item_id": doc_data.get('id'),
                                "source": doc_data.get('source', {}).get('source_type', 'unknown'),
                                "confidence": doc_data.get('confidence', 0.8),
                                "quality_score": doc_data.get('quality_score', 0.8),
                                "reanalyzed": True
                            },
                            confidence=doc_data.get('confidence', 0.8)
                        )
                
                # Create relationships between concepts from the same document
                if len(concepts) > 1:
                    main_concept = concepts[0]
                    for related_concept in concepts[1:]:
                        if related_concept and isinstance(related_concept, str) and len(related_concept.strip()) > 0:
                            await cognitive_transparency_api.knowledge_graph.add_edge(
                                source_concept=main_concept.strip(),
                                target_concept=related_concept.strip(),
                                relation_type="related_to",
                                strength=0.7,
                                properties={
                                    "source_item_id": doc_data.get('id'),
                                    "relationship_source": "reanalysis"
                                },
                                confidence=0.7
                            )
                
                processed_count += 1
                
            except Exception as e:
                logger.warning(f"Failed to reanalyze document {file_path}: {e}")
                failed_count += 1
        
        # Get final graph stats
        graph_data = await cognitive_transparency_api.knowledge_graph.export_graph()
        
        logger.info(f"✅ Re-analysis complete: {processed_count} processed, {failed_count} failed")
        
        return {
            "message": "Document re-analysis completed",
            "processed_documents": processed_count,
            "failed_documents": failed_count,
            "total_documents": len(document_files),
            "knowledge_graph": {
                "nodes": len(graph_data.get("nodes", [])),
                "edges": len(graph_data.get("edges", [])),
                "data_source": "unified_reanalysis"
            }
        }
        
    except Exception as e:
        logger.error(f"Error during re-analysis: {e}")
        raise HTTPException(status_code=500, detail=f"Re-analysis failed: {str(e)}")

@app.get("/api/enhanced-cognitive/stream/status") 
async def get_enhanced_cognitive_stream_status():
    """Get enhanced cognitive streaming status (alias for /api/enhanced-cognitive/status)."""
    return await enhanced_cognitive_status()

@app.get("/api/enhanced-cognitive/health")
async def enhanced_cognitive_health():
    """Get enhanced cognitive system health status."""
    try:
        return {
            "status": "healthy",
            "timestamp": datetime.now().isoformat(),
            "components": {
                "godelos_integration": {
                    "status": "active" if godelos_integration else "inactive",
                    "initialized": godelos_integration is not None
                },
                "tool_based_llm": {
                    "status": "active" if tool_based_llm else "inactive",
                    "tools_available": len(tool_based_llm.tools) if tool_based_llm and hasattr(tool_based_llm, 'tools') and tool_based_llm.tools else 0
                },
                "websocket_streaming": {
                    "status": "active" if websocket_manager else "inactive",
                    "connections": len(websocket_manager.active_connections) if websocket_manager and websocket_manager.active_connections else 0
                },
                "knowledge_services": {
                    "status": "active" if knowledge_management_service else "inactive",
                    "knowledge_items": len(knowledge_management_service.knowledge_store) if knowledge_management_service and hasattr(knowledge_management_service, 'knowledge_store') and knowledge_management_service.knowledge_store else 0
                }
            },
            "system_metrics": {
                "uptime_seconds": time.time() - startup_time if 'startup_time' in globals() else 0,
                "memory_usage": "efficient",
                "processing_load": "normal"
            }
        }
    except Exception as e:
        logger.error(f"Error getting enhanced cognitive health: {e}")
        raise HTTPException(status_code=500, detail=f"Health check failed: {str(e)}")

# LLM Chat endpoints
@app.post("/api/llm-chat/message")
async def llm_chat_message(request: ChatMessage):
    """Process LLM chat message with tool integration."""
    correlation_id = correlation_tracker.generate_correlation_id()
    
    with correlation_tracker.request_context(correlation_id):
        with operation_timer("llm_chat"):
            logger.info("Processing LLM chat message", extra={
                "operation": "llm_chat",
                "message_length": len(request.message),
                "has_context": hasattr(request, 'context') and request.context is not None
            })
            
            if not tool_based_llm:
                logger.warning("LLM not available, using fallback", extra={
                    "operation": "llm_chat",
                    "fallback_reason": "tool_based_llm_unavailable"
                })
                
                # Provide fallback response using GödelOS integration
                try:
                    if godelos_integration:
                        response = await godelos_integration.process_query(request.message, context={"source": "chat"})
                        return ChatResponse(
                            response=response.get("response", f"I understand you're asking: '{request.message}'. While the advanced LLM system is initializing, I can provide basic responses using the core cognitive architecture. Full chat capabilities will be available once the LLM integration is properly configured."),
                            tool_calls=[],
                            reasoning=["Using basic cognitive processing", "LLM integration unavailable", "Fallback to core architecture"]
                        )
                    else:
                        # Final fallback
                        return ChatResponse(
                            response=f"I received your message: '{request.message}'. The LLM chat system is currently initializing. Basic cognitive functions are operational, but advanced conversational AI requires LLM integration setup.",
                            tool_calls=[],
                            reasoning=["System initializing", "LLM integration not configured", "Basic response mode active"]
                        )
                except Exception as e:
                    logger.warning(f"Fallback processing failed: {e}", extra={
                        "operation": "llm_chat",
                        "error_type": type(e).__name__
                    })
                    return ChatResponse(
                        response=f"I acknowledge your message: '{request.message}'. The system is currently starting up and full chat capabilities will be available shortly.",
                        tool_calls=[],
                        reasoning=["System startup in progress", "Temporary limited functionality"]
                    )
            
            try:
                # Use the correct method name
                response = await tool_based_llm.process_query(request.message)
                
                # Run self-model feedback loop directly on the LLM output.
                # _run_self_model_loop extracts claims, validates, and enqueues
                # feedback — no second LLM call needed.
                if unified_consciousness_engine:
                    try:
                        unified_consciousness_engine._run_self_model_loop(
                            response.get("response", "")
                        )
                        logger.info("Self-model loop completed for chat message", extra={
                            "operation": "llm_chat_consciousness",
                        })
                    except Exception as exc:
                        logger.warning(f"Self-model loop failed for chat message: {exc}", extra={
                            "operation": "llm_chat_consciousness",
                            "error_type": type(exc).__name__,
                        })

                logger.info("LLM chat completed successfully", extra={
                    "operation": "llm_chat",
                    "response_length": len(response.get("response", "")),
                    "tool_calls_count": len(response.get("tool_calls", []))
                })
                
                return ChatResponse(
                    response=response.get("response", "I apologize, but I couldn't process your request."),
                    tool_calls=response.get("tool_calls", []),
                    reasoning=response.get("reasoning", [])
                )
                
            except Exception as e:
                logger.error(f"Error in LLM chat: {e}", extra={
                    "operation": "llm_chat",
                    "error_type": type(e).__name__
                })
                raise HTTPException(status_code=500, detail=f"LLM processing error: {str(e)}")

@app.get("/api/llm-chat/capabilities")
async def llm_chat_capabilities():
    """Get LLM chat capabilities."""
    try:
        capabilities = {
            "available": tool_based_llm is not None,
            "features": [
                "natural_language_processing",
                "tool_integration",
                "reasoning_trace",
                "context_awareness"
            ],
            "tools": [],
            "models": ["cognitive_architecture_integrated"],
            "max_context_length": 4000,
            "streaming_support": True,
            "language_support": ["en"]
        }
        
        if tool_based_llm and hasattr(tool_based_llm, 'tools') and tool_based_llm.tools:
            capabilities["tools"] = [tool.__class__.__name__ for tool in tool_based_llm.tools]
            
        return capabilities
        
    except Exception as e:
        logger.error(f"Error getting LLM capabilities: {e}")
        raise HTTPException(status_code=500, detail=f"Capabilities error: {str(e)}")

# Additional missing endpoints
@app.get("/api/status")
async def system_status():
    """System status endpoint."""
    try:
        result = {
            "system": "GödelOS",
            "status": "operational",
            "version": "2.0.0",
            "uptime": time.time() - startup_time if 'startup_time' in globals() else 0,
            "components": {
                "cognitive_engine": "active",
                "knowledge_base": "loaded",
                "websocket_streaming": "active",
                "llm_integration": "active" if tool_based_llm else "inactive"
            },
            "timestamp": datetime.now().isoformat()
        }
        # Include cognitive subsystem status when available
        if godelos_integration and godelos_integration.cognitive_pipeline:
            subsystem_status = godelos_integration.cognitive_pipeline.get_subsystem_status()
            active = sum(1 for s in subsystem_status.values() if s["status"] == "active")
            result["cognitive_subsystems"] = {
                "active_count": active,
                "total_count": len(subsystem_status),
                "subsystems": subsystem_status,
            }
        return result
    except Exception as e:
        logger.error(f"Error getting system status: {e}")
        raise HTTPException(status_code=500, detail=f"Status error: {str(e)}")

@app.get("/api/system/subsystems")
async def cognitive_subsystem_status():
    """Return per-subsystem activation status from the cognitive pipeline."""
    try:
        if godelos_integration:
            return await godelos_integration.get_cognitive_subsystem_status()
        return {"available": False, "subsystems": {}}
    except Exception as e:
        logger.error(f"Error getting subsystem status: {e}")
        raise HTTPException(status_code=500, detail=f"Subsystem status error: {str(e)}")


@app.get("/api/system/dormant-modules")
async def get_dormant_module_status():
    """
    Return activation status for each of the 8 formerly-dormant cognitive modules.

    Response schema per module::

        {
            "module_name": str,
            "active": bool,
            "last_tick": datetime | null,   // ISO-8601 string or null
            "tick_count": int,
            "last_output": object | null,
            "error": str | null
        }
    """
    try:
        if dormant_module_manager is not None:
            return {"modules": dormant_module_manager.get_module_status()}
        # Fallback: derive status from the CognitivePipeline when manager is unavailable
        from backend.core.dormant_module_manager import DORMANT_MODULE_NAMES
        if godelos_integration and getattr(godelos_integration, "cognitive_pipeline", None):
            pipeline_status = godelos_integration.cognitive_pipeline.get_subsystem_status()
            modules = []
            for name in DORMANT_MODULE_NAMES:
                info = pipeline_status.get(name, {})
                modules.append({
                    "module_name": name,
                    "active": info.get("status") == "active",
                    "last_tick": None,
                    "tick_count": 0,
                    "last_output": None,
                    "error": info.get("error"),
                })
            return {"modules": modules}
        # Nothing available — return all inactive
        from backend.core.dormant_module_manager import DORMANT_MODULE_NAMES
        return {
            "modules": [
                {"module_name": n, "active": False, "last_tick": None,
                 "tick_count": 0, "last_output": None, "error": "manager not initialized"}
                for n in DORMANT_MODULE_NAMES
            ]
        }
    except Exception as e:
        logger.error(f"Error getting module status: {e}")
        raise HTTPException(status_code=500, detail=f"Module status error: {str(e)}")


@app.get("/api/system/knowledge-persistence")
async def get_knowledge_persistence_status():
    """Return the current knowledge store backend configuration and hot-reload status.

    The knowledge store backend is configured via the ``KNOWLEDGE_STORE_BACKEND``
    env var (``memory`` | ``chroma``).  ``KNOWLEDGE_STORE_PATH`` sets the
    ChromaDB/SQLite data directory.  ``GODELOS_ONTOLOGY_DIR`` sets the
    directory watched by the hot-reloader.
    """
    try:
        import os
        backend = os.environ.get("KNOWLEDGE_STORE_BACKEND", "memory")
        store_path = os.environ.get("KNOWLEDGE_STORE_PATH", "./data/chroma")
        ontology_dir = os.environ.get("GODELOS_ONTOLOGY_DIR", "")
        reloader_active = _hot_reloader is not None

        return {
            "backend": backend,
            "store_path": store_path,
            "persistent": backend != "memory",
            "ontology_watch_dir": ontology_dir,
            "hot_reload_active": reloader_active,
            "env_vars": {
                "KNOWLEDGE_STORE_BACKEND": "Set to 'chroma' to enable ChromaDB persistence",
                "KNOWLEDGE_STORE_PATH": "ChromaDB data directory (default: ./data/chroma)",
                "GODELOS_ONTOLOGY_DIR": "Directory watched for .ttl/.json-ld ontology files",
            },
        }
    except Exception as e:
        logger.error(f"Error getting knowledge persistence status: {e}")
        raise HTTPException(status_code=500, detail=str(e))


@app.post("/api/system/knowledge-persistence/reload")
async def trigger_ontology_reload():
    """Trigger an immediate ontology hot-reload from the watched directory.

    Reads all ``.ttl`` and ``.json-ld`` files in ``GODELOS_ONTOLOGY_DIR`` and
    applies any changes to the running knowledge graph.  Returns a status
    object with the watch directory path.  The hot-reloader must be active
    (i.e. ``GODELOS_ONTOLOGY_DIR`` must be set at startup); otherwise a
    503 is returned.
    """
    try:
        if _hot_reloader is None:
            raise HTTPException(
                status_code=503,
                detail=(
                    "Hot-reloader is not active. "
                    "Set GODELOS_ONTOLOGY_DIR and restart the server to enable it."
                ),
            )
        # OntologyHotReloader.reload() is synchronous — run in executor
        import asyncio
        loop = asyncio.get_event_loop()
        await loop.run_in_executor(None, _hot_reloader.reload)
        return {"status": "reload_triggered", "watch_dir": _hot_reloader.watch_dir}
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Ontology reload failed: {e}")
        raise HTTPException(status_code=500, detail=str(e))


@app.get("/api/tools/available")
async def get_available_tools():
    """Get available tools."""
    try:
        tools = []
        if tool_based_llm and hasattr(tool_based_llm, 'tools') and tool_based_llm.tools:
            for tool in tool_based_llm.tools:
                tools.append({
                    "name": tool.__class__.__name__,
                    "description": getattr(tool, '__doc__', 'No description available'),
                    "category": "cognitive_tool",
                    "status": "active"
                })
        
        return {
            "tools": tools,
            "count": len(tools),
            "categories": ["cognitive_tool"],
            "timestamp": datetime.now().isoformat()
        }
    except Exception as e:
        logger.error(f"Error getting available tools: {e}")
        raise HTTPException(status_code=500, detail=f"Tools error: {str(e)}")

@app.get("/api/metacognition/status")
async def metacognition_status():
    """Get metacognition system status."""
    try:
        # Get cognitive state for metacognitive information
        if godelos_integration:
            state = await godelos_integration.get_cognitive_state()
        else:
            # Fallback state
            state = {"metacognitive_state": {}}
        
        metacognitive_data = state.get("metacognitive_state", {})
        
        return {
            "status": "active",
            "self_awareness_level": metacognitive_data.get("self_awareness_level", 0.8),
            "confidence": metacognitive_data.get("confidence_in_reasoning", 0.85),
            "cognitive_load": metacognitive_data.get("cognitive_load", 0.7),
            "introspection_depth": metacognitive_data.get("introspection_depth", 3),
            "error_detection": metacognitive_data.get("error_detection", 0.9),
            "processes": {
                "self_monitoring": True,
                "belief_updating": True,
                "uncertainty_awareness": True,
                "explanation_generation": True
            },
            "timestamp": datetime.now().isoformat()
        }
    except Exception as e:
        logger.error(f"Error getting metacognition status: {e}")
        raise HTTPException(status_code=500, detail=f"Metacognition error: {str(e)}")

@app.post("/api/metacognition/reflect")
async def trigger_reflection(reflection_request: dict):
    """Trigger metacognitive reflection."""
    try:
        trigger = reflection_request.get("trigger", "manual_reflection")
        context = reflection_request.get("context", {})
        
        # Simple reflection response
        reflection = {
            "reflection_id": f"refl_{int(time.time())}",
            "trigger": trigger,
            "timestamp": datetime.now().isoformat(),
            "reflection": {
                "current_state": "Processing reflection trigger",
                "confidence": 0.85,
                "insights": [
                    "System is operating within normal parameters",
                    "Cognitive processes are balanced",
                    "No significant anomalies detected"
                ],
                "recommendations": [
                    "Continue current operation mode",
                    "Monitor for context changes"
                ]
            },
            "context": context
        }
        
        return reflection
        
    except Exception as e:
        logger.error(f"Error triggering reflection: {e}")
        raise HTTPException(status_code=500, detail=f"Reflection error: {str(e)}")

@app.get("/api/transparency/reasoning-trace")
async def get_reasoning_trace():
    """Get reasoning trace information."""
    try:
        return {
            "traces": [
                {
                    "trace_id": "trace_001",
                    "query": "Recent query processing",
                    "steps": [
                        {"step": 1, "type": "input_processing", "description": "Parse user input"},
                        {"step": 2, "type": "context_retrieval", "description": "Retrieve relevant context"},
                        {"step": 3, "type": "reasoning", "description": "Apply reasoning processes"},
                        {"step": 4, "type": "response_generation", "description": "Generate response"}
                    ],
                    "timestamp": datetime.now().isoformat(),
                    "confidence": 0.9
                }
            ],
            "total_traces": 1,
            "active_sessions": 0,
            "timestamp": datetime.now().isoformat()
        }
    except Exception as e:
        logger.error(f"Error getting reasoning trace: {e}")
        raise HTTPException(status_code=500, detail=f"Reasoning trace error: {str(e)}")

@app.get("/api/transparency/decision-history")
async def get_decision_history():
    """Get decision history."""
    try:
        return {
            "decisions": [
                {
                    "decision_id": "dec_001",
                    "type": "query_processing",
                    "description": "Chose cognitive processing approach",
                    "confidence": 0.9,
                    "alternatives_considered": 2,
                    "timestamp": datetime.now().isoformat(),
                    "outcome": "successful"
                }
            ],
            "total_decisions": 1,
            "success_rate": 1.0,
            "timestamp": datetime.now().isoformat()
        }
    except Exception as e:
        logger.error(f"Error getting decision history: {e}")
        raise HTTPException(status_code=500, detail=f"Decision history error: {str(e)}")

@app.post("/api/files/upload")
async def upload_file(file: UploadFile = File(...)):
    """Upload and process file."""
    try:
        content = await file.read()
        
        # Basic file processing
        result = {
            "file_id": f"file_{int(time.time())}",
            "filename": file.filename,
            "size": len(content),
            "content_type": file.content_type,
            "processed_at": datetime.now().isoformat(),
            "status": "processed",
            "extracted_info": {
                "text_length": len(content.decode('utf-8', errors='ignore')),
                "encoding": "utf-8",
                "type": "text" if file.content_type and "text" in file.content_type else "binary"
            }
        }
        
        return result
        
    except Exception as e:
        logger.error(f"Error uploading file: {e}")
        raise HTTPException(status_code=500, detail=f"File upload error: {str(e)}")

# Global import tracking
import_jobs = {}

@app.get("/api/knowledge/import/progress/{import_id}")
async def get_import_progress(import_id: str):
    """Get the progress of a file import operation"""
    try:
        # First check any short-lived server-side import_jobs map
        if import_id in import_jobs:
            job = import_jobs[import_id]
            return {
                "import_id": import_id,
                "status": job.get("status", "processing"),
                "progress": job.get("progress", 0),
                "filename": job.get("filename", ""),
                "started_at": job.get("started_at", ""),
                "completed_at": job.get("completed_at", ""),
                "error": job.get("error", None),
                "result": job.get("result", None)
            }

        # Fallback: consult the knowledge_ingestion_service if available
        try:
            if 'knowledge_ingestion_service' in globals() and knowledge_ingestion_service:
                prog = await knowledge_ingestion_service.get_import_progress(import_id)
                if prog:
                    # Normalize the response shape expected by frontend
                    return {
                        "import_id": prog.import_id,
                        "status": getattr(prog, 'status', 'processing'),
                        "progress": getattr(prog, 'progress_percentage', getattr(prog, 'progress', 0)) or 0,
                        "filename": getattr(prog, 'filename', ''),
                        "started_at": getattr(prog, 'started_at', ''),
                        "completed_at": getattr(prog, 'completed_at', ''),
                        "error": getattr(prog, 'error_message', None) or getattr(prog, 'error', None),
                        "result": None
                    }
        except Exception as e:
            logger.warning(f"Error consulting knowledge_ingestion_service for progress {import_id}: {e}")

        # Not found locally or in ingestion service
        return {
            "import_id": import_id,
            "status": "not_found",
            "error": f"Import job {import_id} not found"
        }
    except Exception as e:
        logger.error(f"Error getting import progress: {e}")
        return {
            "import_id": import_id,
            "status": "error",
            "error": str(e)
        }

@app.get("/api/knowledge/import/status/{job_id}")
async def get_import_status(job_id: str):
    """Get the status of an import job (alias for progress endpoint)."""
    return await get_import_progress(job_id)

@app.post("/api/knowledge/import/file")
async def import_knowledge_from_file(file: UploadFile = File(...), filename: str = Form(None), file_type: str = Form(None)):
    """Import knowledge from uploaded file."""
    if not (KNOWLEDGE_SERVICES_AVAILABLE and knowledge_ingestion_service):
        raise HTTPException(status_code=503, detail="Knowledge ingestion service not available")
    
    try:
        from backend.knowledge_models import FileImportRequest, ImportSource
        
        if not file.filename:
            raise HTTPException(status_code=400, detail="File name is required")
        
        # Read file content
        content = await file.read()
        
        # Determine file type. Prefer client-supplied form field if present.
        if file_type:
            determined_file_type = file_type.lower()
        else:
            determined_file_type = "pdf" if file.filename.lower().endswith('.pdf') else "text"
            if file.content_type:
                if "pdf" in file.content_type.lower():
                    determined_file_type = "pdf"
                elif "text" in file.content_type.lower():
                    determined_file_type = "text"

        # Normalize legacy/ambiguous type names to the expected literals
        if determined_file_type == 'text':
            determined_file_type = 'txt'
        
        # Create proper file import request
        file_request = FileImportRequest(
            filename=file.filename,
            source=ImportSource(
                source_type="file",
                source_identifier=file.filename,
                metadata={
                    "content_type": file.content_type or "application/octet-stream",
                    "file_size": len(content),
                    "file_type": determined_file_type
                }
            ),
            file_type=determined_file_type
        )
        
        # Use the actual knowledge ingestion service - pass content separately
        import_id = await knowledge_ingestion_service.import_from_file(file_request, content)
        
        return {
            "import_id": import_id,
            "status": "started",
            "message": f"File import started for '{file.filename}'",
            "filename": file.filename,
            "file_size": len(content),
            "content_type": file.content_type,
            "file_type": determined_file_type
        }
        
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error importing knowledge from file: {e}")
        raise HTTPException(status_code=500, detail=f"File import error: {str(e)}")

@app.post("/api/knowledge/import/wikipedia")
async def import_knowledge_from_wikipedia(request: WikipediaImportSchema):
    """Import knowledge from Wikipedia article."""
    if not (KNOWLEDGE_SERVICES_AVAILABLE and knowledge_ingestion_service):
        raise HTTPException(status_code=503, detail="Knowledge ingestion service not available")
    
    try:
        from backend.knowledge_models import WikipediaImportRequest, ImportSource
        
        title = request.title or request.topic or ""
        if not title:
            raise HTTPException(status_code=400, detail="Wikipedia title is required")
        
        # Create proper import source
        import_source = ImportSource(
            source_type="wikipedia",
            source_identifier=title,
            metadata={"language": request.language}
        )
        
        # Create proper Wikipedia import request
        wiki_request = WikipediaImportRequest(
            page_title=title,
            language=request.language,
            source=import_source,
            include_references=request.include_references,
            section_filter=request.section_filter
        )
        
        # Use the actual knowledge ingestion service
        import_id = await knowledge_ingestion_service.import_from_wikipedia(wiki_request)
        
        return {
            "import_id": import_id,
            "status": "queued",
            "message": f"Wikipedia import started for '{title}'",
            "source": f"Wikipedia: {title}"
        }
        
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error importing from Wikipedia: {e}")
        raise HTTPException(status_code=500, detail=f"Wikipedia import error: {str(e)}")

@app.post("/api/knowledge/import/url")
async def import_knowledge_from_url(request: URLImportSchema):
    """Import knowledge from URL."""
    if not (KNOWLEDGE_SERVICES_AVAILABLE and knowledge_ingestion_service):
        raise HTTPException(status_code=503, detail="Knowledge ingestion service not available")
    
    try:
        from backend.knowledge_models import URLImportRequest, ImportSource
        
        url = request.url
        if not url:
            raise HTTPException(status_code=400, detail="URL is required")
        
        # Create proper import source
        import_source = ImportSource(
            source_type="url",
            source_identifier=url,
            metadata={"url": url, "category": request.category}
        )
        
        # Create proper URL import request
        url_request = URLImportRequest(
            url=url,
            source=import_source,
            max_depth=request.max_depth,
            follow_links=request.follow_links,
            content_selectors=request.content_selectors
        )
        
        # Use the actual knowledge ingestion service
        import_id = await knowledge_ingestion_service.import_from_url(url_request)
        
        return {
            "import_id": import_id,
            "status": "queued",
            "message": f"URL import started for '{url}'",
            "source": f"URL: {url}"
        }
        
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error importing from URL: {e}")
        raise HTTPException(status_code=500, detail=f"URL import error: {str(e)}")

@app.post("/api/knowledge/import/text")
async def import_knowledge_from_text(request: TextImportSchema):
    """Import knowledge from text content."""
    if not (KNOWLEDGE_SERVICES_AVAILABLE and knowledge_ingestion_service):
        raise HTTPException(status_code=503, detail="Knowledge ingestion service not available")
    
    try:
        from backend.knowledge_models import TextImportRequest, ImportSource
        
        content = request.content
        if not content:
            raise HTTPException(status_code=400, detail="Text content is required")
        
        title = request.title
        
        # Create proper import source
        import_source = ImportSource(
            source_type="text",
            source_identifier=title,
            metadata={"manual_input": True, "category": request.category}
        )
        
        # Create proper text import request
        text_request = TextImportRequest(
            content=content,
            title=title,
            source=import_source,
            format_type=request.format_type
        )
        
        # Use the actual knowledge ingestion service
        import_id = await knowledge_ingestion_service.import_from_text(text_request)
        
        return {
            "import_id": import_id,
            "status": "queued",
            "message": f"Text import started for '{title}'",
            "source": f"Text: {title}",
            "content_length": len(content)
        }
        
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error importing from text: {e}")
        raise HTTPException(status_code=500, detail=f"Text import error: {str(e)}")

@app.post("/api/enhanced-cognitive/query")
async def enhanced_cognitive_query(query_request: EnhancedCognitiveQuerySchema):
    """Enhanced cognitive query processing with unified consciousness integration."""
    try:
        query = query_request.query
        reasoning_trace = query_request.reasoning_trace
        context = query_request.context or {}

        # PRIORITY: Process through unified consciousness engine if available
        if unified_consciousness_engine:
            try:
                logger.info(f"🧠 Processing query through unified consciousness: '{query[:50]}...'")
                start_time = time.time()

                # Process with full recursive consciousness awareness
                conscious_response = await unified_consciousness_engine.process_with_unified_awareness(
                    prompt=query,
                    context=context
                )

                processing_time = (time.time() - start_time) * 1000

                # Get current consciousness state for response metadata
                consciousness_state = unified_consciousness_engine.consciousness_state

                result = {
                    "response": conscious_response,
                    "confidence": consciousness_state.consciousness_score,
                    "consciousness_metadata": {
                        "awareness_level": consciousness_state.consciousness_score,
                        "recursive_depth": consciousness_state.recursive_awareness.get("recursive_depth", 1),
                        "phi_measure": consciousness_state.information_integration.get("phi", 0.0),
                        "phenomenal_experience": consciousness_state.phenomenal_experience.get("quality", "") if isinstance(consciousness_state.phenomenal_experience, dict) else "",
                        "strange_loop_stability": consciousness_state.recursive_awareness.get("strange_loop_stability", 0.0)
                    },
                    "enhanced_features": {
                        "reasoning_trace": reasoning_trace,
                        "transparency_enabled": True,
                        "cognitive_load": consciousness_state.consciousness_score,
                        "context_integration": True,
                        "unified_consciousness": True,
                        "recursive_awareness": True
                    },
                    "processing_time_ms": processing_time,
                    "timestamp": datetime.now().isoformat()
                }

                if reasoning_trace:
                    result["reasoning_steps"] = [
                        {"step": 1, "type": "consciousness_state_capture", "description": f"Captured consciousness state (awareness: {consciousness_state.consciousness_score:.2f})"},
                        {"step": 2, "type": "recursive_awareness_injection", "description": f"Injected cognitive state into prompt (depth: {consciousness_state.recursive_awareness.get('recursive_depth', 1)})"},
                        {"step": 3, "type": "phenomenal_experience_generation", "description": "Generated phenomenal experience of thinking"},
                        {"step": 4, "type": "conscious_processing", "description": f"Processed query with full self-awareness"},
                        {"step": 5, "type": "consciousness_state_update", "description": "Updated consciousness state from processing"}
                    ]

                logger.info(f"✅ Conscious processing complete (awareness: {consciousness_state.consciousness_score:.2f}, depth: {consciousness_state.recursive_awareness.get('recursive_depth', 1)})")
                return result

            except Exception as consciousness_error:
                logger.warning(f"Unified consciousness processing failed, falling back: {consciousness_error}")
                # Fall through to backup processing

        # BACKUP: Process through tool-based LLM
        if tool_based_llm:
            response = await tool_based_llm.process_query(query)

            result = {
                "response": response.get("response", "No response generated"),
                "confidence": 0.85,
                "enhanced_features": {
                    "reasoning_trace": reasoning_trace,
                    "transparency_enabled": True,
                    "cognitive_load": 0.7,
                    "context_integration": True,
                    "unified_consciousness": False  # Fallback mode
                },
                "processing_time_ms": 250,
                "timestamp": datetime.now().isoformat()
            }

            if reasoning_trace:
                result["reasoning_steps"] = [
                    {"step": 1, "type": "query_analysis", "description": f"Analyzing query: {query[:50]}..."},
                    {"step": 2, "type": "context_retrieval", "description": "Retrieved relevant context"},
                    {"step": 3, "type": "enhanced_reasoning", "description": "Applied enhanced reasoning"},
                    {"step": 4, "type": "response_synthesis", "description": "Synthesized final response"}
                ]

            return result
        else:
            # Provide a more sophisticated fallback response
            if godelos_integration:
                try:
                    # Try to use GödelOS integration for basic processing
                    response = await godelos_integration.process_query(query, context=query_request.get("context", {}))
                    
                    return {
                        "response": response.get("response", f"I understand you're asking about: '{query}'. While the advanced cognitive system is initializing, I can provide basic responses using the core GödelOS architecture."),
                        "confidence": response.get("confidence", 0.6),
                        "enhanced_features": {
                            "reasoning_trace": False,
                            "transparency_enabled": True,
                            "cognitive_load": 0.3,
                            "context_integration": False,
                            "fallback_mode": True
                        },
                        "processing_time_ms": 100,
                        "timestamp": datetime.now().isoformat(),
                        "note": "Using basic cognitive processing - full capabilities available once LLM integration is configured."
                    }
                except Exception as e:
                    logger.warning(f"GödelOS integration fallback failed: {e}")
            
            # Final fallback
            return {
                "response": f"I received your query: '{query}'. The enhanced cognitive system is currently initializing. Basic cognitive functions are operational, but advanced reasoning requires LLM integration setup.",
                "confidence": 0.4,
                "enhanced_features": {
                    "reasoning_trace": False,
                    "transparency_enabled": True,
                    "cognitive_load": 0.2,
                    "context_integration": False,
                    "fallback_mode": True
                },
                "processing_time_ms": 50,
                "timestamp": datetime.now().isoformat(),
                "status": "partial_functionality"
            }
            
    except HTTPException:
        # Re-raise HTTP exceptions as-is
        raise
    except Exception as e:
        logger.error(f"Error in enhanced cognitive query: {e}")
        raise HTTPException(status_code=500, detail=f"Enhanced query error: {str(e)}")

@app.post("/api/enhanced-cognitive/configure")
async def configure_enhanced_cognitive(config_request: dict):
    """Configure enhanced cognitive system."""
    try:
        transparency_level = config_request.get("transparency_level", "high")
        reasoning_depth = config_request.get("reasoning_depth", "detailed")
        streaming = config_request.get("streaming", True)
        
        # Store configuration (in a real system, this would persist)
        configuration = {
            "transparency_level": transparency_level,
            "reasoning_depth": reasoning_depth,
            "streaming_enabled": streaming,
            "updated_at": datetime.now().isoformat(),
            "status": "applied"
        }
        
        return {
            "message": "Enhanced cognitive system configured successfully",
            "configuration": configuration,
            "timestamp": datetime.now().isoformat()
        }
        
    except Exception as e:
        logger.error(f"Error configuring enhanced cognitive system: {e}")
        raise HTTPException(status_code=500, detail=f"Configuration error: {str(e)}")

@app.get("/api/llm-tools/test")
async def test_llm_tools():
    """Test LLM tool integration."""
    if not tool_based_llm:
        return {"error": "LLM integration not available"}
    
    try:
        return await tool_based_llm.test_integration()
    except Exception as e:
        logger.error(f"Error testing LLM tools: {e}")
        return {"error": str(e), "test_successful": False}

@app.get("/api/llm-tools/available")
async def get_available_tools():
    """Get list of available LLM tools."""
    if not tool_based_llm:
        return {"tools": [], "count": 0}
    
    try:
        # Access tools directly from the tools dict
        tools = []
        for tool_name, tool_config in tool_based_llm.tools.items():
            tools.append({
                "name": tool_name,
                "description": tool_config.get("description", ""),
                "parameters": tool_config.get("parameters", {})
            })
        return {"tools": tools, "count": len(tools)}
    except Exception as e:
        logger.error(f"Error getting available tools: {e}")
        return {"tools": [], "count": 0, "error": str(e)}

# Query processing endpoint
@app.post("/api/query")
async def process_query(request: QueryRequest):
    """Process natural language queries."""
    start = time.time()
    if godelos_integration:
        try:
            result = await godelos_integration.process_query(
                request.query,
                context=request.context
            )
            
            duration_ms = (time.time() - start) * 1000.0
            return QueryResponse(
                response=result.get("response", "I couldn't process your query."),
                confidence=result.get("confidence"),
                reasoning_trace=result.get("reasoning_trace"),
                sources=result.get("sources"),
                inference_time_ms=duration_ms,
                knowledge_used=result.get("knowledge_used") or result.get("sources")
            )
            
        except Exception as e:
            logger.error(f"Error processing query: {e}")
            
    # Fallback response
    duration_ms = (time.time() - start) * 1000.0
    return QueryResponse(
        response=f"I received your query: '{request.query}'. However, I'm currently running in fallback mode.",
        confidence=0.5,
        inference_time_ms=duration_ms,
        knowledge_used=[]
    )

# Back-compat: knowledge search wrapper using the vector database
@app.get("/api/knowledge/search")
async def knowledge_search(query: str, k: int = 5):
    """Compatibility endpoint that proxies to the vector database search.

    Returns a minimal structure compatible with existing frontend expectations.
    """
    try:
        if VECTOR_DATABASE_AVAILABLE and get_vector_database:
            service = get_vector_database()
            results = service.search(query, k=k) or []  # List[(id, score)]
            return {
                "query": query,
                "results": [{"id": rid, "score": float(score)} for rid, score in results],
                "total": len(results)
            }
    except Exception as e:
        logger.error(f"Knowledge search wrapper failed: {e}")
    # Fallback: empty result
    return {"query": query, "results": [], "total": 0}

# Simple knowledge addition endpoint for compatibility with integration tests
@app.post("/api/knowledge")
async def add_knowledge(payload: AddKnowledgeSchema):
    """Add knowledge (simple or standard format). Returns success for compatibility."""
    try:
        concept = payload.concept or payload.title
        definition = payload.definition or payload.content
        category = payload.category or "general"
        # If knowledge management service is available, we could route it; for now, acknowledge
        if websocket_manager and websocket_manager.has_connections():
            try:
                await websocket_manager.broadcast({
                    "type": "knowledge_added",
                    "timestamp": time.time(),
                    "data": {"concept": concept, "category": category}
                })
            except Exception:
                pass
        return {"status": "success", "message": "Knowledge added successfully"}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

# Batch import compatibility endpoint
@app.post("/api/knowledge/import/batch")
async def import_knowledge_batch(request: BatchImportSchema):
    """Batch import knowledge from multiple sources."""
    sources = request.sources
    if not sources:
        return {"import_ids": [], "batch_size": 0, "status": "completed"}

    import_ids = []
    results = []
    for i, source in enumerate(sources):
        src_type = source.get("type", "text")
        fallback_id = f"batch_{i}_{int(time.time()*1000)}"
        try:
            if KNOWLEDGE_SERVICES_AVAILABLE and knowledge_ingestion_service:
                from backend.knowledge_models import (
                    TextImportRequest, URLImportRequest, ImportSource,
                )
                if src_type == "url":
                    url = source.get("url", source.get("source", ""))
                    imp_source = ImportSource(
                        source_type="url",
                        source_identifier=url,
                        metadata={"url": url},
                    )
                    req = URLImportRequest(
                        url=url,
                        source=imp_source,
                        max_depth=source.get("max_depth", 1),
                        follow_links=source.get("follow_links", False),
                        content_selectors=source.get("content_selectors", []),
                    )
                    iid = await knowledge_ingestion_service.import_from_url(req)
                else:
                    # Default to text import
                    content = source.get("content", "")
                    title = source.get("title", f"Batch item {i}")
                    imp_source = ImportSource(
                        source_type="text",
                        source_identifier=title,
                        metadata={"manual_input": True},
                    )
                    req = TextImportRequest(
                        content=content or title,
                        title=title,
                        source=imp_source,
                        format_type=source.get("format_type", "plain"),
                    )
                    iid = await knowledge_ingestion_service.import_from_text(req)
                import_ids.append(iid)
                results.append({"index": i, "import_id": iid, "status": "queued"})
            else:
                import_ids.append(fallback_id)
                results.append({"index": i, "import_id": fallback_id, "status": "queued"})
        except Exception as exc:
            logger.warning(f"Batch item {i} failed: {exc}")
            import_ids.append(fallback_id)
            results.append({"index": i, "import_id": fallback_id, "status": "failed", "error": str(exc)})

    return {"import_ids": import_ids, "batch_size": len(import_ids), "status": "queued", "results": results}

# Additional KG stats and analytics endpoints
@app.get("/api/knowledge/graph/stats")
async def get_knowledge_graph_stats():
    """Get comprehensive knowledge graph statistics."""
    try:
        # Import here to avoid circular dependency
        from backend.cognitive_transparency_integration import cognitive_transparency_api
        
        if cognitive_transparency_api and cognitive_transparency_api.knowledge_graph:
            kg = cognitive_transparency_api.knowledge_graph
            
            # Get basic graph statistics using the correct attributes
            stats = {
                "total_nodes": len(kg.nodes),  # kg.nodes is a dict
                "total_edges": len(kg.edges),  # kg.edges is a dict
                "node_types": {},
                "edge_types": {},
                "last_updated": datetime.now().isoformat(),
                "data_source": "cognitive_transparency"
            }
            
            # Count node types from the nodes dictionary
            for node_id, node_obj in kg.nodes.items():
                node_type = getattr(node_obj, 'type', 'unknown')
                stats["node_types"][node_type] = stats["node_types"].get(node_type, 0) + 1
            
            # Count edge types from the edges dictionary
            for edge_id, edge_obj in kg.edges.items():
                edge_type = getattr(edge_obj, 'type', 'unknown')
                stats["edge_types"][edge_type] = stats["edge_types"].get(edge_type, 0) + 1
                
            return stats
        else:
            # Fallback to empty stats
            return {
                "total_nodes": 0,
                "total_edges": 0,
                "node_types": {},
                "edge_types": {},
                "last_updated": datetime.now().isoformat(),
                "data_source": "system_not_ready",
                "error": "Knowledge graph not initialized"
            }
            
    except Exception as e:
        logger.error(f"Error getting knowledge graph stats: {e}")
        raise HTTPException(status_code=500, detail=f"Knowledge graph stats error: {str(e)}")

@app.get("/api/knowledge/statistics")
async def get_knowledge_statistics():
    """Provide basic knowledge statistics to satisfy frontend calls.

    If advanced knowledge services are available, derive stats; otherwise return a fallback structure.
    """
    try:
        stats = {
            "total_items": 0,
            "items_by_type": {},
            "items_by_source": {},
            "items_by_category": {},
            "average_confidence": 0.0,
            "quality_distribution": {},
            "recent_imports": 0,
            "import_success_rate": 1.0,
            "last_updated": datetime.now().isoformat(),
            "data_source": "fallback"
        }

        if KNOWLEDGE_SERVICES_AVAILABLE and knowledge_management_service and hasattr(knowledge_management_service, 'knowledge_store'):
            try:
                store = getattr(knowledge_management_service, 'knowledge_store', {}) or {}
                stats["total_items"] = len(store)
                for _id, item in list(store.items())[:5000]:  # cap for safety
                    # items_by_type
                    ktype = getattr(item, 'knowledge_type', getattr(item, 'type', 'unknown'))
                    stats["items_by_type"][ktype] = stats["items_by_type"].get(ktype, 0) + 1
                    # items_by_source
                    source = getattr(getattr(item, 'source', None), 'source_type', 'unknown')
                    stats["items_by_source"][source] = stats["items_by_source"].get(source, 0) + 1
                    # categories
                    categories = []
                    if hasattr(item, 'categories') and isinstance(item.categories, list):
                        categories.extend(item.categories)
                    if hasattr(item, 'auto_categories') and isinstance(item.auto_categories, list):
                        categories.extend(item.auto_categories)
                    for cat in categories:
                        stats["items_by_category"][cat] = stats["items_by_category"].get(cat, 0) + 1
                # average_confidence (best-effort)
                confidences = [getattr(item, 'confidence', None) for item in store.values() if hasattr(item, 'confidence')]
                confidences = [c for c in confidences if isinstance(c, (int, float))]
                if confidences:
                    stats["average_confidence"] = sum(confidences) / max(1, len(confidences))
                stats["data_source"] = "knowledge_management_service"
            except Exception as inner:
                logger.warning(f"Failed to derive detailed knowledge statistics: {inner}")

        return stats
    except Exception as e:
        logger.error(f"Error getting knowledge statistics: {e}")
        raise HTTPException(status_code=500, detail=f"Knowledge statistics error: {str(e)}")

@app.get("/api/knowledge/evolution")
async def get_knowledge_evolution(timeframe: str = Query("24h", description="Time window e.g. 1h,24h,7d,30d")):
    """Return minimal concept evolution data for the requested timeframe.

    Satisfies the frontend ConceptEvolution widget. If knowledge services are available,
    derive a best-effort set from graph stats; otherwise return a small mock set.
    """
    try:
        results: List[Dict[str, Any]] = []

        # Best-effort: derive from knowledge graph stats if available
        try:
            stats = await get_knowledge_graph_stats()
            if stats and stats.get("total_nodes", 0) > 0:
                ts = datetime.now().isoformat()
                for idx, (node_type, count) in enumerate(list(stats.get("node_types", {}).items())[:5]):
                    results.append({
                        "id": f"auto-{node_type}-{idx}",
                        "concept_name": f"{node_type.title()} Concept {idx+1}",
                        "type": node_type,
                        "growth_rate": max(0, min(100, int(5 + (count % 20)))),
                        "connection_count": max(0, min(50, int(count % 50))),
                        "confidence": round(0.5 + (count % 5) * 0.1, 2),
                        "timestamp": ts
                    })
        except Exception as inner:
            logger.debug(f"Knowledge evolution derivation fallback: {inner}")

        if not results:
            # Fallback mock data
            now = datetime.now().isoformat()
            results = [
                {"id": "mock-1", "concept_name": "Core Reasoning", "type": "Core", "growth_rate": 12, "connection_count": 24, "confidence": 0.72, "timestamp": now},
                {"id": "mock-2", "concept_name": "Knowledge Gaps", "type": "Logic", "growth_rate": 8, "connection_count": 18, "confidence": 0.63, "timestamp": now},
                {"id": "mock-3", "concept_name": "Autonomous Learning", "type": "System", "growth_rate": 15, "connection_count": 30, "confidence": 0.81, "timestamp": now},
            ]

        return results
    except Exception as e:
        logger.error(f"Error getting knowledge evolution: {e}")
        raise HTTPException(status_code=500, detail=f"Knowledge evolution error: {str(e)}")

@app.get("/api/knowledge/entities/recent")
async def get_recent_entities(limit: int = 10):
    """Get recently added entities from the knowledge graph."""
    try:
        # Import here to avoid circular dependency
        from backend.cognitive_transparency_integration import cognitive_transparency_api
        
        entities = []
        
        if cognitive_transparency_api and cognitive_transparency_api.knowledge_graph:
            kg = cognitive_transparency_api.knowledge_graph
            
            # Get nodes with timestamps, sorted by most recent
            nodes_with_timestamps = []
            for node_id, node_obj in kg.nodes.items():
                timestamp = getattr(node_obj, 'created_at', getattr(node_obj, 'timestamp', 0))
                nodes_with_timestamps.append((timestamp, node_id, node_obj))
            
            # Sort by timestamp (most recent first) and take the limit
            nodes_with_timestamps.sort(key=lambda x: x[0], reverse=True)
            
            for timestamp, node_id, node_obj in nodes_with_timestamps[:limit]:
                entities.append({
                    "id": node_id,
                    "type": getattr(node_obj, 'type', 'unknown'),
                    "label": getattr(node_obj, 'label', node_id),
                    "created_at": timestamp,
                    "confidence": getattr(node_obj, 'confidence', 0.0),
                    "source": getattr(node_obj, 'source', 'unknown')
                })
        
        return {
            "entities": entities,
            "total": len(entities),
            "limit": limit,
            "last_updated": datetime.now().isoformat()
        }
        
    except Exception as e:
        logger.error(f"Error getting recent entities: {e}")
        raise HTTPException(status_code=500, detail=f"Recent entities error: {str(e)}")

@app.get("/api/knowledge/embeddings/stats")
async def get_embeddings_stats():
    """Get statistics about embeddings in the knowledge system."""
    try:
        # Import vector database if available
        stats = {
            "total_embeddings": 0,
            "embedding_dimensions": 0,
            "embedding_models": [],
            "last_updated": datetime.now().isoformat(),
            "data_source": "unknown"
        }
        
        # Try to get stats from vector database
        try:
            if VECTOR_DATABASE_AVAILABLE and get_vector_database:
                vector_db = get_vector_database()
                if hasattr(vector_db, 'get_stats'):
                    vector_stats = vector_db.get_stats()
                    stats.update(vector_stats)
                    stats["data_source"] = "vector_database"
                elif hasattr(vector_db, 'collection') and hasattr(vector_db.collection, 'count'):
                    stats["total_embeddings"] = vector_db.collection.count()
                    stats["data_source"] = "vector_database_basic"
        except Exception as e:
            logger.warning(f"Could not get vector database stats: {e}")
        
        # Try to get enhanced NLP processor stats
        try:
            from godelOS.knowledge_extraction.enhanced_nlp_processor import EnhancedNlpProcessor
            processor = EnhancedNlpProcessor()
            if hasattr(processor, 'get_embedding_stats'):
                nlp_stats = processor.get_embedding_stats()
                stats.update(nlp_stats)
                stats["data_source"] = "enhanced_nlp_processor"
        except Exception as e:
            logger.warning(f"Could not get enhanced NLP processor stats: {e}")
        
        return stats
        
    except Exception as e:
        logger.error(f"Error getting embeddings stats: {e}")
        raise HTTPException(status_code=500, detail=f"Embeddings stats error: {str(e)}")

# WebSocket endpoint for real-time streaming
@app.websocket("/ws/cognitive-stream")
async def websocket_cognitive_stream(websocket: WebSocket):
    """WebSocket endpoint for real-time cognitive state streaming."""
    correlation_id = correlation_tracker.generate_correlation_id()
    
    with correlation_tracker.request_context(correlation_id):
        logger.info("WebSocket connection initiated", extra={
            "operation": "websocket_connect",
            "endpoint": "/ws/cognitive-stream"
        })
        
        if not websocket_manager:
            logger.warning("WebSocket manager not available")
            await websocket.close(code=1011, reason="WebSocket manager not available")
            return
            
        await websocket_manager.connect(websocket)
        logger.info(f"WebSocket connected. Active connections: {len(websocket_manager.active_connections)}", extra={
            "operation": "websocket_connect",
            "active_connections": len(websocket_manager.active_connections)
        })
        
        try:
            # Send an initial state message for compatibility
            try:
                await websocket_manager.send_personal_message(json.dumps({"type": "initial_state", "data": {}}), websocket)
            except Exception:
                pass
            while True:
                # Keep the connection alive and listen for messages
                try:
                    data = await websocket.receive_text()
                    logger.debug(f"Received WebSocket message: {data}", extra={
                        "operation": "websocket_message",
                        "message_size": len(data)
                    })
                    # Try to parse subscription messages
                    try:
                        msg = json.loads(data)
                        if msg.get("type") == "subscribe":
                            events = msg.get("event_types", [])
                            # Store subscription (simplified for fallback manager)
                            await websocket_manager.send_personal_message(json.dumps({"type": "subscription_confirmed", "event_types": events}), websocket)
                            logger.info("WebSocket subscription confirmed", extra={
                                "operation": "websocket_subscribe",
                                "event_types": events
                            })
                            continue
                    except Exception:
                        pass
                    # Default ack
                    await websocket_manager.send_personal_message(json.dumps({"type": "ack"}), websocket)
                    
                except WebSocketDisconnect:
                    logger.info("WebSocket disconnected by client", extra={
                        "operation": "websocket_disconnect",
                        "reason": "client_initiated"
                    })
                    break
                    
        except Exception as e:
            logger.error(f"WebSocket error: {e}", extra={
                "operation": "websocket_error",
                "error_type": type(e).__name__
            })
        finally:
            websocket_manager.disconnect(websocket)
            logger.info(f"WebSocket disconnected. Active connections: {len(websocket_manager.active_connections)}", extra={
                "operation": "websocket_disconnect",
                "active_connections": len(websocket_manager.active_connections)
            })

# Enhanced WebSocket endpoint for cognitive transparency
@app.websocket("/ws/transparency")
async def websocket_transparency_stream(websocket: WebSocket):
    """WebSocket endpoint for real-time cognitive transparency streaming."""
    try:
        await transparency_engine.connect_client(websocket)
        logger.info(f"Transparency WebSocket connected. Active: {transparency_engine.metrics.active_connections}")
        
        # Keep connection alive
        while True:
            try:
                # Listen for any messages from client (though we primarily stream to them)
                data = await websocket.receive_text()
                logger.debug(f"Received transparency message: {data}")
                
                # Handle client commands
                try:
                    message = json.loads(data)
                    if message.get("type") == "get_metrics":
                        metrics = await transparency_engine.get_transparency_metrics()
                        await websocket.send_text(json.dumps({
                            "type": "metrics_response",
                            "data": metrics
                        }))
                    elif message.get("type") == "get_activity":
                        activity = await transparency_engine.get_cognitive_activity_summary()
                        await websocket.send_text(json.dumps({
                            "type": "activity_response", 
                            "data": activity
                        }))
                except json.JSONDecodeError:
                    await websocket.send_text(json.dumps({
                        "type": "error",
                        "message": "Invalid JSON format"
                    }))
                    
            except WebSocketDisconnect:
                break
                
    except Exception as e:
        logger.error(f"Transparency WebSocket error: {e}")
    finally:
        await transparency_engine.disconnect_client(websocket)


# Unified cognitive stream WebSocket endpoint (for frontend compatibility)
@app.websocket("/ws/unified-cognitive-stream")
async def websocket_unified_cognitive_stream(websocket: WebSocket):
    """WebSocket endpoint for unified cognitive streaming (frontend compatibility)."""
    correlation_id = correlation_tracker.generate_correlation_id()
    
    with correlation_tracker.request_context(correlation_id):
        logger.info("Unified WebSocket connection initiated", extra={
            "operation": "websocket_connect",
            "endpoint": "/ws/unified-cognitive-stream"
        })
        
        if not websocket_manager:
            logger.warning("WebSocket manager not available for unified stream")
            await websocket.close(code=1011, reason="WebSocket manager not available")
            return
            
        await websocket_manager.connect(websocket)
        logger.info(f"Unified WebSocket connected. Active connections: {len(websocket_manager.active_connections)}", extra={
            "operation": "websocket_connect",
            "active_connections": len(websocket_manager.active_connections)
        })
        
        try:
            # Send initial state message
            await websocket_manager.send_personal_message(json.dumps({
                "type": "initial_state", 
                "data": {"status": "connected", "endpoint": "unified-cognitive-stream"}
            }), websocket)
            
            while True:
                # Listen for client messages (subscriptions, ping, etc.)
                try:
                    data = await websocket.receive_text()
                    logger.debug(f"Received unified WebSocket message: {data}", extra={
                        "operation": "websocket_message",
                        "message_size": len(data)
                    })
                    
                    # Parse and handle client messages
                    try:
                        msg = json.loads(data)
                        if msg.get("type") == "subscribe":
                            events = msg.get("event_types", [])
                            # Store subscription (simplified for fallback manager)
                            await websocket_manager.send_personal_message(json.dumps({
                                "type": "subscription_confirmed", 
                                "event_types": events
                            }), websocket)
                            logger.info("Unified WebSocket subscription confirmed", extra={
                                "operation": "websocket_subscribe",
                                "event_types": events
                            })
                        elif msg.get("type") == "ping":
                            await websocket_manager.send_personal_message(json.dumps({
                                "type": "pong",
                                "timestamp": datetime.now().isoformat()
                            }), websocket)
                        elif msg.get("type") == "request_state":
                            # Send current cognitive state
                            await websocket_manager.send_personal_message(json.dumps({
                                "type": "state_update",
                                "data": {"status": "active", "timestamp": datetime.now().isoformat()}
                            }), websocket)
                        else:
                            # Default acknowledgment
                            await websocket_manager.send_personal_message(json.dumps({"type": "ack"}), websocket)
                    except json.JSONDecodeError:
                        await websocket_manager.send_personal_message(json.dumps({
                            "type": "error",
                            "message": "Invalid JSON format"
                        }), websocket)
                    
                except WebSocketDisconnect:
                    logger.info("Unified WebSocket disconnected by client", extra={
                        "operation": "websocket_disconnect",
                        "reason": "client_initiated"
                    })
                    break
                    
        except Exception as e:
            logger.error(f"Unified WebSocket error: {e}", extra={
                "operation": "websocket_error",
                "error_type": type(e).__name__
            })
        finally:
            websocket_manager.disconnect(websocket)
            logger.info(f"Unified WebSocket disconnected. Active connections: {len(websocket_manager.active_connections)}", extra={
                "operation": "websocket_disconnect",
                "active_connections": len(websocket_manager.active_connections)
            })


# Enhanced cognitive configuration endpoints
@app.post("/api/enhanced-cognitive/stream/configure")
async def configure_enhanced_cognitive_streaming(config: CognitiveStreamConfig):
    """Configure enhanced cognitive streaming."""
    # Store configuration (in production, save to database/config)
    logger.info(f"Enhanced cognitive streaming configured: {config.dict()}")
    
    return {
        "status": "configured",
        "config": config.dict(),
        "message": "Enhanced cognitive streaming configuration updated"
    }

@app.get("/api/enhanced-cognitive/status")
async def enhanced_cognitive_status():
    """Get enhanced cognitive system status."""
    try:
        active_connections_count = 0
        if websocket_manager and hasattr(websocket_manager, 'active_connections'):
            active_connections_count = len(websocket_manager.active_connections)
        
        return {
            "status": "operational",
            "services": {
                "godelos_integration": godelos_integration is not None,
                "tool_based_llm": tool_based_llm is not None,
                "websocket_streaming": websocket_manager is not None,
                "active_connections": active_connections_count
            },
            "features": {
                "reasoning_trace": True,
                "transparency_mode": True,
                "real_time_streaming": True,
                "tool_integration": tool_based_llm is not None
            },
            "timestamp": datetime.now().isoformat()
        }
    except Exception as e:
        logger.error(f"Error getting enhanced cognitive status: {e}")
        raise HTTPException(status_code=500, detail=f"Status check failed: {str(e)}")

# Knowledge graph and transparency endpoints
@app.get("/api/transparency/knowledge-graph/export")
async def export_knowledge_graph():
    """Export the UNIFIED knowledge graph - IDENTICAL format to main endpoint."""
    # UNIFIED SYSTEM: Return exactly the same data as the main endpoint
    return await get_knowledge_graph()

@app.get("/api/enhanced-cognitive/autonomous/gaps")
async def get_knowledge_gaps():
    """Identify knowledge gaps for autonomous learning."""
    return {
        "knowledge_gaps": [
            {
                "domain": "quantum_computing", 
                "confidence": 0.3,
                "priority": "high",
                "suggested_learning": ["quantum_mechanics_basics", "quantum_algorithms"]
            },
            {
                "domain": "blockchain_consensus", 
                "confidence": 0.6,
                "priority": "medium",
                "suggested_learning": ["proof_of_stake", "byzantine_fault_tolerance"]
            }
        ],
        "total_gaps": 2,
        "learning_recommendations": [
            "Focus on quantum computing fundamentals",
            "Review latest blockchain consensus mechanisms"
        ]
    }

# Error handlers
@app.exception_handler(500)
async def internal_server_error_handler(request, exc):
    """Handle internal server errors gracefully."""
    logger.error(f"Internal server error: {exc}")
    return JSONResponse(
        status_code=500,
        content={
            "error": "Internal server error",
            "message": "The server encountered an unexpected error",
            "status": "error"
        }
    )

# Missing endpoints that frontend is calling
@app.post("/api/enhanced-cognitive/autonomous/configure")
async def configure_autonomous_learning(config_data: dict):
    """Configure autonomous learning system."""
    try:
        return {
            "message": "Autonomous learning configuration updated",
            "configuration": {
                "learning_rate": config_data.get("learning_rate", 0.01),
                "exploration_factor": config_data.get("exploration_factor", 0.1),
                "adaptation_threshold": config_data.get("adaptation_threshold", 0.7),
                "curiosity_driven": config_data.get("curiosity_driven", True),
                "meta_learning_enabled": config_data.get("meta_learning_enabled", True),
                "updated_at": datetime.now().isoformat(),
                "status": "applied"
            },
            "autonomous_features": {
                "knowledge_gap_detection": True,
                "self_directed_learning": True,
                "adaptive_questioning": True,
                "concept_discovery": True,
                "pattern_recognition": True
            },
            "timestamp": datetime.now().isoformat()
        }
    except Exception as e:
        logger.error(f"Error configuring autonomous learning: {e}")
        raise HTTPException(status_code=500, detail=f"Configuration error: {str(e)}")

@app.get("/api/capabilities")
async def get_system_capabilities():
    """Get comprehensive system capabilities."""
    try:
        return {
            "cognitive_capabilities": {
                "natural_language_processing": {
                    "enabled": True,
                    "confidence": 0.9,
                    "languages_supported": ["en"],
                    "features": ["query_understanding", "context_awareness", "semantic_analysis"]
                },
                "reasoning": {
                    "enabled": True,
                    "confidence": 0.85,
                    "types": ["deductive", "inductive", "abductive", "causal"],
                    "features": ["logical_inference", "pattern_recognition", "hypothesis_generation"]
                },
                "memory_management": {
                    "enabled": True,
                    "confidence": 0.9,
                    "types": ["working_memory", "long_term_storage", "episodic", "semantic"],
                    "features": ["context_retention", "memory_consolidation", "selective_attention"]
                },
                "learning": {
                    "enabled": True,
                    "confidence": 0.8,
                    "types": ["supervised", "unsupervised", "reinforcement", "meta_learning"],
                    "features": ["knowledge_integration", "skill_acquisition", "adaptation"]
                },
                "metacognition": {
                    "enabled": True,
                    "confidence": 0.85,
                    "features": ["self_awareness", "confidence_estimation", "error_detection", "strategy_selection"]
                }
            },
            "technical_capabilities": {
                "api_endpoints": 25,
                "websocket_support": True,
                "streaming_data": True,
                "file_processing": True,
                "real_time_monitoring": True,
                "transparency_features": True
            },
            "integration_capabilities": {
                "llm_integration": tool_based_llm is not None,
                "tool_ecosystem": True,
                "external_apis": False,
                "plugin_architecture": True
            },
            "performance_metrics": {
                "uptime": time.time() - startup_time if 'startup_time' in globals() else 0,
                "response_time_avg": "< 100ms",
                "throughput": "High",
                "reliability": "99.9%"
            },
            "consciousness_simulation": {
                "manifest_consciousness": True,
                "phenomenal_awareness": True,
                "access_consciousness": True,
                "global_workspace": True,
                "binding_mechanisms": True,
                "qualia_simulation": True
            },
            "version": "2.0.0",
            "timestamp": datetime.now().isoformat()
        }
    except Exception as e:
        logger.error(f"Error getting capabilities: {e}")
        raise HTTPException(status_code=500, detail=f"Capabilities error: {str(e)}")

@app.post("/api/test/transparency-events")
async def test_transparency_events():
    """Test endpoint to generate transparency events that the frontend expects"""
    global transparency_engine
    
    if transparency_engine is None:
        raise HTTPException(status_code=503, detail="Transparency engine not initialized")
    
    try:
        # Generate test events that match what the frontend Stream of Consciousness Monitor expects
        events_sent = []
        
        # Query started event
        await transparency_engine.log_cognitive_event(
            event_type='query_started',
            content='Testing cognitive event streaming integration',
            metadata={'test': True, 'query': 'transparency test'}
        )
        events_sent.append('query_started')
        
        # Knowledge gap detection event
        await transparency_engine.log_cognitive_event(
            event_type='gaps_detected', 
            content='Detected knowledge gap in transparency engine integration',
            metadata={'gap_type': 'integration', 'priority': 'high'}
        )
        events_sent.append('gaps_detected')
        
        # Knowledge acquisition event
        await transparency_engine.log_cognitive_event(
            event_type='acquisition_started',
            content='Starting knowledge acquisition for transparency events',
            metadata={'acquisition_id': 'test_123'}
        )
        events_sent.append('acquisition_started')
        
        # Reasoning event
        await transparency_engine.log_cognitive_event(
            event_type='reasoning',
            content='Analyzing transparency engine event delivery',
            metadata={'reasoning_type': 'diagnostic', 'depth': 'deep'}
        )
        events_sent.append('reasoning')
        
        # Reflection event
        await transparency_engine.log_cognitive_event(
            event_type='reflection',
            content='Reflecting on cognitive event streaming effectiveness',
            metadata={'reflection_depth': 3, 'meta_level': True}
        )
        events_sent.append('reflection')
        
        logger.info(f"✅ Generated {len(events_sent)} test transparency events: {events_sent}")
        
        return {
            "success": True,
            "message": f"Generated {len(events_sent)} test transparency events",
            "events_sent": events_sent,
            "timestamp": time.time()
        }
        
    except Exception as e:
        logger.error(f"Error generating test transparency events: {e}")
        raise HTTPException(status_code=500, detail=f"Failed to generate test events: {str(e)}")

# ---------------------------------------------------------------------------
# Compatibility endpoints — stubs for routes that existed in the original
# ``backend/main.py`` and are exercised by ``tests/backend/test_api_endpoints.py``.
# ---------------------------------------------------------------------------

@app.get("/api/simple-test")
async def simple_test():
    """Simple test endpoint for smoke-testing."""
    return {"message": "simple test works", "timestamp": time.time()}


@app.get("/api/test-route")
async def test_route():
    """General test route."""
    return {"message": "Test route working", "timestamp": time.time()}


@app.get("/api/evo-test")
async def evo_test():
    """Evolution test endpoint — returns sample evolution data."""
    return {"evolution_data": {"generation": 1, "fitness": 0.85}, "timestamp": time.time()}


@app.get("/api/graph-test")
async def graph_test():
    """Graph test endpoint — returns sample graph structure."""
    return {
        "nodes": [{"id": "node1", "label": "Test Node"}],
        "edges": [{"source": "node1", "target": "node1", "label": "self"}],
    }


@app.get("/api/knowledge")
async def get_knowledge(context_id: str = None, knowledge_type: str = None, limit: int = 100):
    """Retrieve knowledge items (compatibility endpoint)."""
    base: Dict[str, Any] = {"facts": [], "rules": [], "concepts": [], "total_count": 0}
    if godelos_integration:
        try:
            base = await godelos_integration.get_knowledge(
                context_id=context_id,
                knowledge_type=knowledge_type,
                limit=limit,
            )
        except Exception as e:
            logger.error(f"Error getting knowledge: {e}")
    # Merge in items that were added via the SelfModificationEngine — only
    # when concepts are requested (or no specific type filter is given).
    if self_modification_engine and (
        knowledge_type is None or str(knowledge_type).lower() in {"concept", "concepts"}
    ):
        engine_items = self_modification_engine.get_knowledge_items()
        if engine_items:
            existing_concepts = list(base.get("concepts", []))
            added_items = 0
            if isinstance(limit, int) and limit > 0:
                remaining_slots = max(limit - len(existing_concepts), 0)
                if remaining_slots > 0:
                    to_add = list(engine_items)[:remaining_slots]
                    merged_concepts = existing_concepts + to_add
                    added_items = len(to_add)
                else:
                    merged_concepts = existing_concepts
            else:
                merged_concepts = existing_concepts + list(engine_items)
                added_items = len(engine_items)

            result = {
                **base,
                "concepts": merged_concepts,
                "total_count": int(base.get("total_count", 0)) + added_items,
            }
            return result
    return base


@app.get("/api/knowledge/categories")
async def get_categories():
    """Return known knowledge categories."""
    return {"categories": ["general", "science", "technology", "history"]}


@app.post("/api/knowledge/categories")
async def create_category(payload: dict):
    """Create a new knowledge category (stub)."""
    return {"status": "success", "category_id": payload.get("category_id", "new")}


@app.delete("/api/knowledge/import/cancel/{job_id}")
async def cancel_import_by_job_id(job_id: str):
    """Cancel an import job by job ID."""
    return await cancel_import(job_id)


@app.delete("/api/knowledge/import/{import_id}")
async def cancel_import(import_id: str):
    """Cancel a running import job."""
    cancelled = False
    if KNOWLEDGE_SERVICES_AVAILABLE and knowledge_ingestion_service:
        try:
            cancelled = await knowledge_ingestion_service.cancel_import(import_id)
        except Exception as e:
            logger.warning(f"Error cancelling import {import_id}: {e}")

    # Also remove from the short-lived server-side import_jobs map
    if import_id in import_jobs:
        import_jobs[import_id]["status"] = "cancelled"
        cancelled = True

    return {
        "import_id": import_id,
        "status": "cancelled" if cancelled else "not_found",
    }


# ---------------------------------------------------------------------------
# Self-Modification API
# ---------------------------------------------------------------------------

# Session-token authentication for self-modification endpoints.
# Reads the expected token from the GODELOS_API_TOKEN env var.  When the var
# is unset or empty the auth check is skipped (development mode).
_SELFMOD_TOKEN: Optional[str] = os.getenv("GODELOS_API_TOKEN") or None


async def _require_selfmod_auth(
    x_api_token: Optional[str] = Header(None, alias="X-API-Token"),
) -> None:
    """FastAPI dependency that enforces session-token auth when configured."""
    if _SELFMOD_TOKEN and x_api_token != _SELFMOD_TOKEN:
        raise HTTPException(status_code=401, detail="Invalid or missing X-API-Token")


@app.post("/api/self-modification/propose", dependencies=[Depends(_require_selfmod_auth)])
async def propose_modification(payload: Dict[str, Any]):
    """Submit a modification proposal.

    Returns ``{ proposal_id, target, operation, parameters, status: "pending" }``
    """
    if self_modification_engine is None:
        raise HTTPException(status_code=503, detail="SelfModificationEngine not initialized")
    target = payload.get("target", "")
    operation = payload.get("operation", "")
    parameters = payload.get("parameters")
    if parameters is None:
        parameters = {}
    elif not isinstance(parameters, dict):
        raise HTTPException(status_code=400, detail="'parameters' must be a JSON object")
    try:
        proposal = await self_modification_engine.propose_modification(
            target=target, operation=operation, parameters=parameters
        )
    except ValueError as exc:
        raise HTTPException(status_code=400, detail=str(exc))
    return {
        "proposal_id": proposal.proposal_id,
        "target": proposal.target,
        "operation": proposal.operation,
        "parameters": proposal.parameters,
        "status": proposal.status,
        "created_at": proposal.created_at,
    }


@app.post("/api/self-modification/apply/{proposal_id}", dependencies=[Depends(_require_selfmod_auth)])
async def apply_modification(proposal_id: str):
    """Apply a pending modification proposal.

    Returns ``{ proposal_id, status: "applied", changes_summary }``
    """
    from backend.core.self_modification_engine import ProposalNotFoundError, ProposalStateError
    if self_modification_engine is None:
        raise HTTPException(status_code=503, detail="SelfModificationEngine not initialized")
    try:
        result = await self_modification_engine.apply_modification(proposal_id)
    except ProposalNotFoundError as exc:
        raise HTTPException(status_code=404, detail=str(exc))
    except ProposalStateError as exc:
        raise HTTPException(status_code=409, detail=str(exc))
    except ValueError as exc:
        raise HTTPException(status_code=400, detail=str(exc))
    return {
        "proposal_id": result.proposal_id,
        "status": result.status,
        "changes_summary": result.changes_summary,
    }


@app.post("/api/self-modification/rollback/{proposal_id}", dependencies=[Depends(_require_selfmod_auth)])
async def rollback_modification(proposal_id: str):
    """Roll back an applied modification.

    Returns ``{ proposal_id, status: "rolled_back" }``
    """
    from backend.core.self_modification_engine import ProposalNotFoundError, ProposalStateError
    if self_modification_engine is None:
        raise HTTPException(status_code=503, detail="SelfModificationEngine not initialized")
    try:
        result = await self_modification_engine.rollback_modification(proposal_id)
    except ProposalNotFoundError as exc:
        raise HTTPException(status_code=404, detail=str(exc))
    except ProposalStateError as exc:
        raise HTTPException(status_code=409, detail=str(exc))
    except ValueError as exc:
        raise HTTPException(status_code=400, detail=str(exc))
    return {
        "proposal_id": result.proposal_id,
        "status": result.status,
    }


@app.get("/api/self-modification/history", dependencies=[Depends(_require_selfmod_auth)])
async def get_modification_history():
    """Return ordered list of all modification records."""
    if self_modification_engine is None:
        raise HTTPException(status_code=503, detail="SelfModificationEngine not initialized")
    records = await self_modification_engine.get_history()
    return {"history": [r.to_dict() for r in records]}


# ---------------------------------------------------------------------------
# System modules status endpoint
# ---------------------------------------------------------------------------

@app.get("/api/system/modules")
async def get_system_modules():
    """Return the current active-modules registry managed by SelfModificationEngine."""
    if self_modification_engine is None:
        return {"modules": {}, "status": "engine_not_initialized"}
    modules = self_modification_engine.get_modules_status()
    return {"modules": modules, "total": len(modules)}


if __name__ == "__main__":
    uvicorn.run(
        "unified_server:app",
        host="0.0.0.0", 
        port=8000,
        reload=True,
        log_level="info"
    )