evaluate.py

#!/usr/bin/env python3
"""End-to-end model evaluation and comparison.

Compares Isolation Forest, Fixed Threshold, LOF, and One-Class SVM
against synthetic ground truth.  Saves results to data/reports/.
"""
import argparse
import sys
from pathlib import Path

from src.utils import setup_logger, ensure_dirs
from src.storage import load_csv
from src.evaluation import run_full_evaluation

logger = setup_logger("evaluate")

RAW_DIR = Path("data/raw")
PROCESSED_DIR = Path("data/processed")
MODEL_DIR = Path("models")
REPORTS_DIR = Path("data/reports")
ensure_dirs([RAW_DIR, PROCESSED_DIR, MODEL_DIR, REPORTS_DIR])


def print_table(results: list[dict]):
    if not results:
        print("No results to display.")
        return
    header = f"{'Model':<25} {'Prec':<8} {'Recall':<8} {'F1':<8} {'Acc':<8} {'FP':<5} {'FN':<5} {'Train(s)':<10} {'Infer(s)':<10}"
    sep = "-" * len(header)
    print(sep)
    print(header)
    print(sep)
    for r in results:
        name = r.get("model_name", "?")
        prec = r.get("precision", 0)
        recall = r.get("recall", 0)
        f1 = r.get("f1", 0)
        acc = r.get("accuracy", 0)
        fp = r.get("false_positive", 0)
        fn = r.get("false_negative", 0)
        train_t = r.get("training_time_s", 0)
        infer_t = r.get("inference_time_s", 0)
        print(f"{name:<25} {prec:<8.4f} {recall:<8.4f} {f1:<8.4f} {acc:<8.4f} {fp:<5} {fn:<5} {train_t:<10.4f} {infer_t:<10.4f}")
    print(sep)


def main():
    parser = argparse.ArgumentParser(description="Evaluate and compare anomaly detection models")
    parser.add_argument("--raw", type=str, default="process_metrics.csv", help="Raw telemetry CSV")
    parser.add_argument("--features", type=str, default="features.csv", help="Features CSV")
    parser.add_argument("--contamination", type=float, default=0.1, help="Expected anomaly proportion")
    parser.add_argument("--cpu-threshold", type=float, default=80.0, help="Fixed CPU threshold")
    parser.add_argument("--mem-threshold", type=float, default=800.0, help="Fixed memory threshold (MB)")
    args = parser.parse_args()

    raw_path = RAW_DIR / args.raw
    features_path = PROCESSED_DIR / args.features

    if not raw_path.exists():
        logger.error("Raw data not found at %s", raw_path)
        logger.info("Run `python run_agent.py --synthetic --duration 30` first")
        sys.exit(1)

    raw = load_csv(raw_path)
    if raw is not None and "is_anomaly_ground_truth" not in raw.columns:
        logger.warning("Ground truth labels not found. Regenerate with updated synthetic module.")
        logger.info("Run: python run_agent.py --synthetic --duration 30")

    logger.info("Starting evaluation (contamination=%.2f, cpu_threshold=%.0f, mem_threshold=%.0f)",
                args.contamination, args.cpu_threshold, args.mem_threshold)

    result = run_full_evaluation(
        raw_path=raw_path,
        features_path=features_path,
        reports_dir=REPORTS_DIR,
        contamination=args.contamination,
        cpu_threshold=args.cpu_threshold,
        mem_threshold=args.mem_threshold,
    )

    error = result.get("error")
    if error:
        logger.error("Evaluation failed: %s", error)
        sys.exit(1)

    results = result.get("results", [])
    summary = result.get("summary", {})

    print()
    print("=" * 80)
    print("MODEL EVALUATION RESULTS")
    print("=" * 80)
    print_table(results)
    print()

    if summary:
        print(f"Best model: {summary.get('best_model', 'N/A')} (F1 = {summary.get('best_f1', 0):.4f})")
    print(f"Reports saved to: {REPORTS_DIR}")
    print("=" * 80)


if __name__ == "__main__":
    main()