cache_manager.py

"""
Multi-level caching strategy for email validation.
Maximizes cache hit rates and minimizes external calls.
"""
import redis
import hashlib
import time
from typing import Optional, Dict, List, Any
from dataclasses import dataclass
from functools import lru_cache
import json
import logging

logger = logging.getLogger(__name__)


@dataclass
class CacheConfig:
    """Cache TTL configuration."""
    mx_records: int = 3600        # 1 hour - MX records change rarely
    smtp_results: int = 1800      # 30 minutes - SMTP results semi-volatile
    disposable: int = 86400       # 24 hours - Disposable domains stable
    syntax: int = -1              # Forever - Syntax doesn't change
    rate_limit: int = 60          # 1 minute - Rate limit tracking
    domain_info: int = 86400      # 24 hours - Domain information


class MultiLevelCache:
    """
    Multi-level caching system:
    
    Level 1: In-memory LRU cache (fastest, smallest)
    Level 2: Local Redis (fast, moderate size)
    Level 3: Shared Redis Cluster (slower, largest)
    """
    
    def __init__(
        self,
        redis_host: str = 'localhost',
        redis_port: int = 6379,
        redis_db: int = 0,
        redis_password: Optional[str] = None,
        lru_size: int = 10000,
        enable_redis: bool = True
    ):
        self.config = CacheConfig()
        
        # Level 2/3: Redis
        self.enable_redis = enable_redis
        self.redis = None
        
        if enable_redis:
            try:
                self.redis = redis.Redis(
                    host=redis_host,
                    port=redis_port,
                    db=redis_db,
                    password=redis_password,
                    decode_responses=True,
                    socket_connect_timeout=2,
                    socket_timeout=2,
                    max_connections=50,
                    health_check_interval=30,
                )
                # Test connection
                self.redis.ping()
                logger.info("Connected to Redis cache")
            except Exception as e:
                logger.warning(f"Could not connect to Redis: {e}")
                self.redis = None
                self.enable_redis = False
        
        # Level 1: In-memory LRU cache
        self._lru_size = lru_size
        self._lru_cache: Dict[str, Any] = {}
        self._lru_access_times: Dict[str, float] = {}
        
        # Cache statistics
        self.stats = {
            'lru_hits': 0,
            'redis_hits': 0,
            'misses': 0,
            'sets': 0,
            'evictions': 0,
        }
    
    def _lru_get(self, key: str) -> Optional[Any]:
        """Get from LRU cache with access time update."""
        if key in self._lru_cache:
            self._lru_access_times[key] = time.time()
            return self._lru_cache[key]
        return None
    
    def _lru_set(self, key: str, value: Any):
        """Set in LRU cache with eviction if needed."""
        # Evict oldest if at capacity
        while len(self._lru_cache) >= self._lru_size:
            if self._lru_access_times:
                oldest_key = min(self._lru_access_times, key=self._lru_access_times.get)
                del self._lru_cache[oldest_key]
                del self._lru_access_times[oldest_key]
                self.stats['evictions'] += 1
            else:
                break
        
        self._lru_cache[key] = value
        self._lru_access_times[key] = time.time()
    
    def _lru_delete(self, key: str):
        """Delete from LRU cache."""
        if key in self._lru_cache:
            del self._lru_cache[key]
            del self._lru_access_times[key]
    
    def get_mx_records(self, domain: str) -> Optional[List[str]]:
        """
        Get cached MX records with multi-level lookup.
        
        Order: LRU -> Redis -> None
        """
        cache_key = f"mx:{domain.lower()}"
        
        # Level 1: Check LRU
        cached = self._lru_get(cache_key)
        if cached is not None:
            self.stats['lru_hits'] += 1
            return cached
        
        # Level 2: Check Redis
        if self.redis:
            try:
                cached = self.redis.get(cache_key)
                if cached:
                    self.stats['redis_hits'] += 1
                    records = json.loads(cached)
                    # Promote to LRU
                    self._lru_set(cache_key, records)
                    return records
            except redis.RedisError as e:
                logger.debug(f"Redis error: {e}")
        
        self.stats['misses'] += 1
        return None
    
    def set_mx_records(self, domain: str, records: List[str]):
        """Cache MX records at all levels."""
        cache_key = f"mx:{domain.lower()}"
        
        # Level 1: LRU
        self._lru_set(cache_key, records)
        
        # Level 2: Redis
        if self.redis:
            try:
                self.redis.setex(
                    cache_key,
                    self.config.mx_records,
                    json.dumps(records)
                )
                self.stats['sets'] += 1
            except redis.RedisError as e:
                logger.debug(f"Redis error: {e}")
    
    def get_smtp_result(self, email: str) -> Optional[Dict]:
        """Get cached SMTP validation result."""
        cache_key = f"smtp:{self._hash_email(email)}"
        
        # Level 1: LRU
        cached = self._lru_get(cache_key)
        if cached is not None:
            self.stats['lru_hits'] += 1
            return cached
        
        # Level 2: Redis
        if self.redis:
            try:
                cached = self.redis.get(cache_key)
                if cached:
                    self.stats['redis_hits'] += 1
                    result = json.loads(cached)
                    self._lru_set(cache_key, result)
                    return result
            except redis.RedisError:
                pass
        
        self.stats['misses'] += 1
        return None
    
    def set_smtp_result(self, email: str, result: Dict):
        """Cache SMTP validation result."""
        cache_key = f"smtp:{self._hash_email(email)}"
        
        # Level 1: LRU
        self._lru_set(cache_key, result)
        
        # Level 2: Redis
        if self.redis:
            try:
                self.redis.setex(
                    cache_key,
                    self.config.smtp_results,
                    json.dumps(result)
                )
                self.stats['sets'] += 1
            except redis.RedisError:
                pass
    
    def is_disposable(self, domain: str) -> Optional[bool]:
        """Check if domain is disposable (cached)."""
        cache_key = f"disposable:{domain.lower()}"
        
        # Check LRU first
        cached = self._lru_get(cache_key)
        if cached is not None:
            self.stats['lru_hits'] += 1
            return cached
        
        # Check Redis
        if self.redis:
            try:
                cached = self.redis.get(cache_key)
                if cached is not None:
                    self.stats['redis_hits'] += 1
                    result = cached == '1'
                    self._lru_set(cache_key, result)
                    return result
            except redis.RedisError:
                pass
        
        self.stats['misses'] += 1
        return None
    
    def set_disposable(self, domain: str, is_disposable: bool):
        """Cache disposable domain check."""
        cache_key = f"disposable:{domain.lower()}"
        
        self._lru_set(cache_key, is_disposable)
        
        if self.redis:
            try:
                self.redis.setex(
                    cache_key,
                    self.config.disposable,
                    '1' if is_disposable else '0'
                )
            except redis.RedisError:
                pass
    
    def get_domain_info(self, domain: str) -> Optional[Dict]:
        """Get cached domain information."""
        cache_key = f"domain_info:{domain.lower()}"
        
        # Check LRU
        cached = self._lru_get(cache_key)
        if cached is not None:
            self.stats['lru_hits'] += 1
            return cached
        
        # Check Redis
        if self.redis:
            try:
                cached = self.redis.get(cache_key)
                if cached:
                    self.stats['redis_hits'] += 1
                    info = json.loads(cached)
                    self._lru_set(cache_key, info)
                    return info
            except redis.RedisError:
                pass
        
        self.stats['misses'] += 1
        return None
    
    def set_domain_info(self, domain: str, info: Dict):
        """Cache domain information."""
        cache_key = f"domain_info:{domain.lower()}"
        
        self._lru_set(cache_key, info)
        
        if self.redis:
            try:
                self.redis.setex(
                    cache_key,
                    self.config.domain_info,
                    json.dumps(info)
                )
            except redis.RedisError:
                pass
    
    def _hash_email(self, email: str) -> str:
        """Create consistent hash for email."""
        return hashlib.sha256(email.lower().encode()).hexdigest()[:16]
    
    def get_stats(self) -> Dict[str, Any]:
        """Get cache statistics."""
        total_requests = self.stats['lru_hits'] + self.stats['redis_hits'] + self.stats['misses']
        hit_rate = 0
        if total_requests > 0:
            hit_rate = (self.stats['lru_hits'] + self.stats['redis_hits']) / total_requests
        
        return {
            **self.stats,
            'hit_rate': hit_rate,
            'lru_size': len(self._lru_cache),
            'total_requests': total_requests,
        }
    
    def clear(self):
        """Clear all caches."""
        self._lru_cache.clear()
        self._lru_access_times.clear()
        
        if self.redis:
            try:
                # Only clear our keys
                for pattern in ['mx:*', 'smtp:*', 'disposable:*', 'domain_info:*']:
                    keys = self.redis.keys(pattern)
                    if keys:
                        self.redis.delete(*keys)
            except redis.RedisError as e:
                logger.error(f"Error clearing Redis cache: {e}")
    
    def warmup(self, domains: List[str], mx_records_map: Dict[str, List[str]]):
        """
        Warm up cache with known MX records.
        Useful for common domains like gmail.com, yahoo.com, etc.
        """
        for domain in domains:
            if domain in mx_records_map:
                self.set_mx_records(domain, mx_records_map[domain])
        
        logger.info(f"Cache warmed up with {len(domains)} domains")


# Cache TTL Recommendations
CACHE_TTL_RECOMMENDATIONS = {
    # MX Records: 1 hour
    # - MX records change very rarely
    # - Reduces DNS queries by ~70%
    'mx_records': 3600,
    
    # SMTP Results: 30 minutes
    # - Mailbox status can change
    # - Balance between accuracy and performance
    'smtp_results': 1800,
    
    # Disposable Domains: 24 hours
    # - Disposable domain lists are relatively stable
    # - Update daily from GitHub sources
    'disposable_domains': 86400,
    
    # Syntax Validation: Permanent
    # - Email syntax doesn't change
    # - Can cache indefinitely
    'syntax': None,
    
    # Rate Limit Tracking: 1 minute
    # - Short TTL for rate limit windows
    'rate_limit': 60,
    
    # Domain Information: 24 hours
    # - Provider info, rate limits, etc.
    'domain_info': 86400,
}


# Common domain MX records for cache warmup
COMMON_MX_RECORDS = {
    'gmail.com': [
        'gmail-smtp-in.l.google.com',
        'alt1.gmail-smtp-in.l.google.com',
        'alt2.gmail-smtp-in.l.google.com',
        'alt3.gmail-smtp-in.l.google.com',
        'alt4.gmail-smtp-in.l.google.com',
    ],
    'googlemail.com': [
        'gmail-smtp-in.l.google.com',
        'alt1.gmail-smtp-in.l.google.com',
        'alt2.gmail-smtp-in.l.google.com',
    ],
    'yahoo.com': [
        'mta5.am0.yahoodns.net',
        'mta6.am0.yahoodns.net',
        'mta7.am0.yahoodns.net',
    ],
    'ymail.com': [
        'mta5.am0.yahoodns.net',
        'mta6.am0.yahoodns.net',
        'mta7.am0.yahoodns.net',
    ],
    'outlook.com': [
        'outlook-com.olc.protection.outlook.com',
    ],
    'hotmail.com': [
        'hotmail-com.olc.protection.outlook.com',
    ],
    'live.com': [
        'live-com.olc.protection.outlook.com',
    ],
    'aol.com': [
        'mx-aol.mail.gm0.yahoodns.net',
    ],
    'icloud.com': [
        'mx01.mail.icloud.com',
        'mx02.mail.icloud.com',
        'mx03.mail.icloud.com',
        'mx04.mail.icloud.com',
        'mx05.mail.icloud.com',
        'mx06.mail.icloud.com',
    ],
    'me.com': [
        'mx01.mail.icloud.com',
        'mx02.mail.icloud.com',
    ],
    'protonmail.com': [
        'mail.protonmail.ch',
        'mailsec.protonmail.ch',
    ],
    'zoho.com': [
        'mx.zoho.com',
        'mx2.zoho.com',
        'mx3.zoho.com',
    ],
}


def create_warmed_cache(redis_url: Optional[str] = None) -> MultiLevelCache:
    """Create a cache pre-warmed with common domain MX records."""
    cache = MultiLevelCache()
    cache.warmup(list(COMMON_MX_RECORDS.keys()), COMMON_MX_RECORDS)
    return cache