various fix on ml pipeline, ui, models, backed

Author: ent0n29

.gitignore

@@ -128,3 +128,6 @@ test-output/
 coverage/
 .coverage
 htmlcov/
+
+# Persistent data (local dev)
+/data/

Makefile

@@ -26,33 +26,42 @@ help: ## Show this help
 	@echo ''
 	@echo '${GREEN}AWARE Fund${RESET} - The Vanguard of Prediction Markets'
 	@echo ''
-	@echo '${YELLOW}Local Development:${RESET}'
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | grep -E 'local|up|down|build|logs|status|clean' | awk 'BEGIN {FS = ":.*?## "}; {printf "  ${BLUE}%-15s${RESET} %s\n", $$1, $$2}'
+	@echo '${YELLOW}Quick Start:${RESET}'
+	@echo '  ${BLUE}make build${RESET}       Rebuild Docker images (after code changes)'
+	@echo '  ${BLUE}make local${RESET}       Start everything (services + analytics)'
+	@echo '  ${BLUE}make down${RESET}        Stop everything'
 	@echo ''
-	@echo '${YELLOW}Server Deployment:${RESET}'
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | grep -E 'deploy|server' | awk 'BEGIN {FS = ":.*?## "}; {printf "  ${BLUE}%-15s${RESET} %s\n", $$1, $$2}'
+	@echo '${YELLOW}ML Training:${RESET}'
+	@echo '  ${BLUE}make train${RESET}       Quick (10K traders, ~8 min)'
+	@echo '  ${BLUE}make train-all${RESET}   All eligible traders (currently ~4K)'
 	@echo ''
-	@echo '${YELLOW}Development:${RESET}'
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | grep -E 'test|lint|java|python' | awk 'BEGIN {FS = ":.*?## "}; {printf "  ${BLUE}%-15s${RESET} %s\n", $$1, $$2}'
+	@echo '${YELLOW}Other:${RESET}'
+	@echo '  ${BLUE}make analytics${RESET}   Run analytics only'
+	@echo '  ${BLUE}make logs${RESET}        View logs'
+	@echo '  ${BLUE}make status${RESET}      Health check'
 	@echo ''
 
 # ═══════════════════════════════════════════════════════════════════════════════
 # LOCAL DEVELOPMENT
 # ═══════════════════════════════════════════════════════════════════════════════
 
-local: docker-check ## Start full local stack (all services)
+local: docker-check ## Start everything + run analytics (one command!)
 	@echo '${GREEN}Starting AWARE Fund local stack...${RESET}'
 	docker compose -f docker-compose.local.yaml up -d
 	@echo ''
-	@echo '${GREEN}Services starting. Access:${RESET}'
+	@echo '${YELLOW}Waiting for services to be ready...${RESET}'
+	@sleep 10
+	@echo '${GREEN}Running analytics pipeline...${RESET}'
+	@docker exec aware-analytics python3 run_all.py 2>&1 | grep -E "(INFO|WARNING|Complete)" | tail -20 || true
+	@echo ''
+	@echo '${GREEN}✓ AWARE Fund ready!${RESET}'
+	@echo ''
 	@echo '  Web Dashboard:     http://localhost:3000'
 	@echo '  Python API:        http://localhost:8000'
 	@echo '  Strategy Service:  http://localhost:8081'
-	@echo '  Executor Service:  http://localhost:8080'
 	@echo '  ClickHouse:        http://localhost:8123'
 	@echo ''
-	@echo 'Run ${BLUE}make logs${RESET} to view logs'
-	@echo 'Run ${BLUE}make status${RESET} to check health'
+	@echo 'Commands: ${BLUE}make train${RESET} (retrain ML) | ${BLUE}make logs${RESET} | ${BLUE}make down${RESET}'
 
 up: local ## Alias for 'make local'
 
@@ -141,12 +150,42 @@ python-setup: ## Setup Python virtual environments
 	cd aware-fund/services/analytics && python -m venv .venv && .venv/bin/pip install -r requirements.txt
 	cd aware-fund/services/api && python -m venv .venv && .venv/bin/pip install -r requirements.txt
 
-python-analytics: ## Run analytics jobs (requires infra)
+python-analytics: ## Run analytics jobs ONCE (requires infra)
 	cd aware-fund/services/analytics && source .venv/bin/activate && CLICKHOUSE_HOST=localhost python run_all.py
 
+python-analytics-continuous: ## Run ML analytics continuously (hourly updates)
+	@echo '${GREEN}Starting ML analytics pipeline (continuous mode)...${RESET}'
+	cd aware-fund/services/analytics && source .venv/bin/activate && CLICKHOUSE_HOST=localhost python run_all.py --continuous --interval 3600
+
 python-api: ## Start Python API (requires infra)
 	cd aware-fund/services/api && source .venv/bin/activate && CLICKHOUSE_HOST=localhost uvicorn main:app --reload --host 0.0.0.0 --port 8000
 
+# ═══════════════════════════════════════════════════════════════════════════════
+# ML PIPELINE
+# ═══════════════════════════════════════════════════════════════════════════════
+
+train: ## Retrain ML models (quick: 10K traders, ~8 min)
+	@echo '${GREEN}Retraining ML models (quick mode)...${RESET}'
+	@echo 'Training on 10K traders, 50 epochs (~8 min)'
+	@docker exec aware-analytics python3 -m ml.training.train --max-traders 10000 --epochs 50
+	@echo ''
+	@echo '${GREEN}Running analytics with new model...${RESET}'
+	@docker exec aware-analytics python3 run_all.py 2>&1 | grep -E "(INFO|Complete)" | tail -10
+	@echo '${GREEN}✓ Training complete! Refresh UI to see changes.${RESET}'
+
+train-all: ## Retrain on ALL eligible traders (no limit)
+	@echo '${GREEN}Retraining ML models on ALL data...${RESET}'
+	@echo 'Training on all eligible traders, 100 epochs'
+	@docker exec aware-analytics python3 -m ml.training.train --max-traders 999999 --epochs 100
+	@echo ''
+	@echo '${GREEN}Running analytics with new model...${RESET}'
+	@docker exec aware-analytics python3 run_all.py 2>&1 | grep -E "(INFO|Complete)" | tail -10
+	@echo '${GREEN}✓ Training complete! Refresh UI to see changes.${RESET}'
+
+analytics: ## Run analytics pipeline only (no training)
+	@echo '${GREEN}Running analytics...${RESET}'
+	@docker exec aware-analytics python3 run_all.py 2>&1 | grep -E "(INFO|WARNING|Complete)" | tail -20
+
 # ═══════════════════════════════════════════════════════════════════════════════
 # WEB DASHBOARD
 # ═══════════════════════════════════════════════════════════════════════════════

analytics-service/clickhouse/init/205_ml_explainability.sql

@@ -0,0 +1,176 @@
+-- ============================================================================
+-- AWARE ML Explainability Schema
+-- ============================================================================
+-- Stores ML model metadata, feature importance, tier boundaries, and training history
+-- for transparency, monitoring, and debugging.
+-- ============================================================================
+
+-- ----------------------------------------------------------------------------
+-- Feature Importance
+-- ----------------------------------------------------------------------------
+-- Stores feature importance scores from XGBoost and other models.
+-- Updated after each training run.
+
+CREATE TABLE IF NOT EXISTS polybot.aware_ml_feature_importance (
+    feature_name        String,
+    importance_score    Float32,
+    importance_rank     UInt16,
+    model_version       LowCardinality(String),
+    importance_type     LowCardinality(String) DEFAULT 'weight',  -- 'weight', 'gain', 'cover'
+    calculated_at       DateTime64(3) DEFAULT now64(3)
+)
+ENGINE = ReplacingMergeTree(calculated_at)
+ORDER BY (model_version, feature_name)
+SETTINGS index_granularity = 8192;
+
+
+-- ----------------------------------------------------------------------------
+-- Tier Boundaries
+-- ----------------------------------------------------------------------------
+-- Documents the score ranges for each tier (BRONZE, SILVER, GOLD, DIAMOND).
+-- Used for explainability and consistency checks.
+
+CREATE TABLE IF NOT EXISTS polybot.aware_ml_tier_boundaries (
+    tier_name           LowCardinality(String),  -- BRONZE, SILVER, GOLD, DIAMOND
+    tier_order          UInt8,                   -- 1, 2, 3, 4 for sorting
+    score_min           Float32,
+    score_max           Float32,
+    confidence_threshold Float32 DEFAULT 0.5,
+    description         String,
+    model_version       LowCardinality(String),
+    updated_at          DateTime64(3) DEFAULT now64(3)
+)
+ENGINE = ReplacingMergeTree(updated_at)
+ORDER BY (model_version, tier_order)
+SETTINGS index_granularity = 8192;
+
+
+-- ----------------------------------------------------------------------------
+-- Training Runs
+-- ----------------------------------------------------------------------------
+-- Logs each model training run for tracking and rollback capability.
+
+CREATE TABLE IF NOT EXISTS polybot.aware_ml_training_runs (
+    run_id              UUID DEFAULT generateUUIDv4(),
+    model_version       LowCardinality(String),
+    started_at          DateTime64(3),
+    completed_at        DateTime64(3),
+    duration_seconds    UInt32,
+    status              LowCardinality(String) DEFAULT 'running',  -- 'running', 'success', 'failed', 'rolled_back'
+
+    -- Training data stats
+    n_traders           UInt32,
+    n_trades            UInt32,
+    train_split_ratio   Float32,
+
+    -- Final metrics
+    tier_accuracy       Float32,
+    sharpe_mae          Float32,
+    val_loss            Float32,
+
+    -- Trigger info
+    trigger_reason      LowCardinality(String),  -- 'scheduled', 'drift', 'manual'
+    triggered_by        String DEFAULT 'system',
+
+    -- Hyperparameters (JSON blob)
+    hyperparameters     String,
+
+    -- Notes
+    notes               String DEFAULT ''
+)
+ENGINE = MergeTree()
+ORDER BY (started_at)
+PARTITION BY toYYYYMM(started_at)
+SETTINGS index_granularity = 8192;
+
+
+-- ----------------------------------------------------------------------------
+-- Drift Reports
+-- ----------------------------------------------------------------------------
+-- Stores drift detection results for monitoring.
+
+CREATE TABLE IF NOT EXISTS polybot.aware_ml_drift_reports (
+    report_id           UUID DEFAULT generateUUIDv4(),
+    checked_at          DateTime64(3) DEFAULT now64(3),
+
+    -- Summary
+    alert_level         LowCardinality(String),  -- 'normal', 'warning', 'critical'
+    drift_ratio         Float32,                 -- Fraction of features that drifted
+    n_features          UInt16,
+    n_drifted           UInt16,
+
+    -- Baseline info
+    baseline_date       DateTime64(3),
+    n_samples_baseline  UInt32,
+    n_samples_current   UInt32,
+
+    -- Action taken
+    retrain_triggered   UInt8 DEFAULT 0,
+    retrain_reason      String DEFAULT '',
+
+    -- Detailed results (JSON array)
+    feature_results     String
+)
+ENGINE = MergeTree()
+ORDER BY (checked_at)
+PARTITION BY toYYYYMM(checked_at)
+SETTINGS index_granularity = 8192;
+
+
+-- ============================================================================
+-- Seed Data: Default Tier Boundaries
+-- ============================================================================
+
+INSERT INTO polybot.aware_ml_tier_boundaries
+(tier_name, tier_order, score_min, score_max, confidence_threshold, description, model_version)
+VALUES
+('BRONZE', 1, 0.0, 49.9, 0.3, 'Entry-level traders. May have limited history or inconsistent performance.', 'ensemble_v1'),
+('SILVER', 2, 50.0, 69.9, 0.4, 'Developing traders. Show some edge but not consistently profitable.', 'ensemble_v1'),
+('GOLD', 3, 70.0, 89.9, 0.5, 'Skilled traders. Consistent profitability with good risk management.', 'ensemble_v1'),
+('DIAMOND', 4, 90.0, 100.0, 0.7, 'Elite traders. Top performers with exceptional track records.', 'ensemble_v1');
+
+
+-- ============================================================================
+-- Views
+-- ============================================================================
+
+-- Latest feature importance
+CREATE VIEW IF NOT EXISTS polybot.v_ml_feature_importance_latest AS
+SELECT
+    feature_name,
+    importance_score,
+    importance_rank,
+    model_version,
+    importance_type
+FROM polybot.aware_ml_feature_importance FINAL
+WHERE model_version = (
+    SELECT model_version
+    FROM polybot.aware_ml_feature_importance
+    ORDER BY calculated_at DESC
+    LIMIT 1
+)
+ORDER BY importance_rank;
+
+
+-- Latest training run
+CREATE VIEW IF NOT EXISTS polybot.v_ml_training_latest AS
+SELECT *
+FROM polybot.aware_ml_training_runs
+WHERE status = 'success'
+ORDER BY completed_at DESC
+LIMIT 1;
+
+
+-- Drift trend (last 30 days)
+CREATE VIEW IF NOT EXISTS polybot.v_ml_drift_trend AS
+SELECT
+    toDate(checked_at) AS date,
+    avg(drift_ratio) AS avg_drift_ratio,
+    max(drift_ratio) AS max_drift_ratio,
+    countIf(alert_level = 'warning') AS warning_count,
+    countIf(alert_level = 'critical') AS critical_count,
+    countIf(retrain_triggered = 1) AS retrains_triggered
+FROM polybot.aware_ml_drift_reports
+WHERE checked_at >= now() - INTERVAL 30 DAY
+GROUP BY date
+ORDER BY date;

analytics-service/clickhouse/init/300_user_investments.sql

@@ -227,6 +227,8 @@ INSERT INTO polybot.aware_fund_summary
 VALUES
 ('PSI-10', 'active', 0, 0, 1.0, 0, 0, 0, 0, 0, 0.005, 0.10, 10,
  'Top 10 Smart Money traders by score. Mirrors their positions proportionally.', today()),
+('PSI-25', 'active', 0, 0, 1.0, 0, 0, 0, 0, 0, 0.005, 0.10, 10,
+ 'Top 25 Smart Money traders by score. Broader diversification than PSI-10.', today()),
 ('PSI-SPORTS', 'active', 0, 0, 1.0, 0, 0, 0, 0, 0, 0.005, 0.10, 10,
  'Top sports betting specialists. Focus on NFL, NBA, Soccer markets.', today()),
 ('PSI-CRYPTO', 'active', 0, 0, 1.0, 0, 0, 0, 0, 0, 0.005, 0.10, 10,

aware-fund/services/analytics/Dockerfile

@@ -2,12 +2,19 @@ FROM python:3.11-slim
 
 WORKDIR /app
 
-# Install dependencies
+# Install system dependencies for ML
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gcc \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Python dependencies
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
 # Copy source code
 COPY *.py .
+COPY ml/ ./ml/
+COPY notifications/ ./notifications/
 
 # Environment defaults
 ENV CLICKHOUSE_HOST=clickhouse
@@ -17,6 +24,7 @@ ENV SCORING_INTERVAL_SECONDS=3600
 ENV MIN_TRADES=10
 ENV MAX_TRADERS=10000
 ENV LOG_LEVEL=INFO
+ENV PYTHONUNBUFFERED=1
 
-# Default: run orchestrator
-CMD ["python", "run_all.py"]
+# Default: run ML analytics continuously (hourly scoring + drift monitoring)
+CMD ["python", "run_all.py", "--continuous", "--interval", "3600"]

aware-fund/services/analytics/anomaly_detection.py

@@ -216,11 +216,10 @@ def get_integrity_score(self, username: str) -> IntegrityScore:
     def _get_traders_to_scan(self) -> list[str]:
         """Get traders with sufficient activity to analyze"""
         query = """
-        SELECT DISTINCT username
+        SELECT DISTINCT proxy_address
         FROM polybot.aware_global_trades
         WHERE ts >= now() - INTERVAL 30 DAY
-          AND username != ''
-        GROUP BY username
+        GROUP BY proxy_address
         HAVING count() >= 20
         LIMIT 5000
         """

aware-fund/services/analytics/edge_persistence.py

@@ -229,11 +229,10 @@ def predict(self, username: str) -> Optional[PersistencePrediction]:
     def _get_eligible_traders(self) -> list[str]:
         """Get traders eligible for prediction"""
         query = f"""
-        SELECT DISTINCT username
+        SELECT DISTINCT proxy_address
         FROM polybot.aware_global_trades
         WHERE ts >= now() - INTERVAL 90 DAY
-          AND username != ''
-        GROUP BY username
+        GROUP BY proxy_address
         HAVING count() >= {self.config.min_trades}
         LIMIT 2000
         """

aware-fund/services/analytics/hidden_alpha.py

@@ -151,7 +151,6 @@ def find_hidden_gems(self) -> list[HiddenTrader]:
                 sharpe_ratio >= {self.config.min_sharpe_for_gem}
                 AND total_volume_usd <= {self.config.max_volume_for_hidden}
                 AND total_trades >= {self.config.min_trades_for_gem}
-                AND username != ''
         ),
         -- Calculate percentile ranks for quality metrics
         ranked AS (
@@ -293,7 +292,6 @@ def find_rising_stars(self) -> list[HiddenTrader]:
                 AND win_rate >= {self.config.min_win_rate_star}
                 AND sharpe_ratio >= {self.config.min_sharpe_star}
                 AND total_trades >= 10
-                AND username != ''
         ),
         -- Calculate performance relative to tenure
         performance_adjusted AS (
@@ -453,7 +451,6 @@ def find_niche_specialists(self) -> list[HiddenTrader]:
             unique_markets <= 5
             AND total_trades >= 20
             AND sharpe_ratio >= 1.0
-            AND username != ''
         ORDER BY sharpe_ratio DESC
         LIMIT {self.config.max_discoveries_per_type}
         """
@@ -523,7 +520,6 @@ def find_contrarians(self) -> list[HiddenTrader]:
             AND total_pnl > 0
             AND sharpe_ratio >= 0.5
             AND total_trades >= 20
-            AND username != ''
         ORDER BY total_pnl DESC
         LIMIT {self.config.max_discoveries_per_type}
         """