Project (1).py

#!/usr/bin/env python
# coding: utf-8

# In[2]:


from mpl_toolkits.mplot3d import Axes3D
from sklearn.preprocessing import StandardScaler
import matplotlib.pyplot as plt # plotting
import numpy as np # linear algebra
import os # accessing directory structure
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, confusion_matrix
import warnings
warnings.filterwarnings('ignore')

# In[9]:


accepted_raw = pd.read_csv('accepted_2007_to_2018Q4.csv')

print(f"Original shape: {accepted_raw.shape}")
print("First few columns:", accepted_raw.columns[:10].tolist())
print("\nSample issue_d:", accepted_raw['issue_d'].head().tolist() if 'issue_d' in accepted_raw.columns else "No issue_d")
print("Sample loan_status:", accepted_raw['loan_status'].head().tolist() if 'loan_status' in accepted_raw.columns else "No loan_status")


# In[ ]:


rejected = pd.read_csv('rejected_2007_to_2018Q4.csv')

print(f"Rejected shape: {rejected.shape}")
print("First few columns:", rejected.columns[:10].tolist())



# In[3]:
# Keep 70% complete columns
accepted_clean = accepted_raw.dropna(axis=1, thresh=len(accepted_raw)*0.7)
print(f"After gentle clean: {accepted_clean.shape}")

# Keep only essential rows (has loan_status)
accepted_clean = accepted_clean.dropna(subset=['loan_status'])
print(f"After loan_status filter: {accepted_clean.shape}")

# In[4]:
# STEP 3: Binary target + random split
accepted_clean = accepted_clean.copy()
accepted_clean['loan_status_binary'] = accepted_clean['loan_status'].apply(
    lambda x: 1 if x in ['Charged Off', 'Default'] else 0
)

train, test = train_test_split(
    accepted_clean,
    test_size=0.3,
    random_state=42,
    stratify=accepted_clean['loan_status_binary']
)

# PURE NUMERIC FEATURES ONLY (no strings)
numeric_features = ['loan_amnt', 'int_rate', 'annual_inc', 'dti']
numeric_features = [f for f in numeric_features if f in train.columns]

# Convert to numeric + fill NaNs
X_train = train[numeric_features].apply(pd.to_numeric, errors='coerce').fillna(0)
y_train = train['loan_status_binary']
X_test = test[numeric_features].apply(pd.to_numeric, errors='coerce').fillna(0)
y_test = test['loan_status_binary']

# Train
rf = RandomForestClassifier(n_estimators=50, random_state=42, n_jobs=-1)
rf.fit(X_train, y_train)

y_pred = rf.predict(X_test)
model_accuracy = accuracy_score(y_test, y_pred)
print(f"🎯 Test Accuracy: {model_accuracy:.1%}")

# STATE FAIRNESS (keep addr_state separate for analysis)
print("\n🏛️ STATE FPRs (Top/Bottom 5):")
state_fpr = {}

for state in test['addr_state'].value_counts().head(10).index:
    if test['addr_state'].eq(state).sum() > 1000:
        mask = test['addr_state'] == state
        cm = confusion_matrix(y_test[mask], y_pred[mask], labels=[0, 1])
        fpr = cm[0, 1] / (cm[0, 1] + cm[0, 0]) if (cm[0, 1] + cm[0, 0]) > 0 else 0
        state_fpr[state] = fpr

for state, fpr in sorted(state_fpr.items(), key=lambda x: x[1], reverse=True)[:5]:
    print(f"{state}: {fpr:.1%}")



# In[13]: TIME-BASED VALIDATION (Production Deployment Simulation)
print("\n⏰ TIME-BASED VALIDATION (2007-2014 TRAIN | 2015-2018 TEST)")
print("="*65)

# Parse issue_d dates and create time split
accepted_clean['issue_date'] = pd.to_datetime(accepted_clean['issue_d'], errors='coerce')
accepted_clean = accepted_clean.dropna(subset=['issue_date'])  # Drop invalid dates

# Split: Train 2007-2014, Test 2015-2018 (realistic deployment)
cutoff_date = '2015-01-01'
train_time = accepted_clean[accepted_clean['issue_date'] < cutoff_date]
test_time = accepted_clean[accepted_clean['issue_date'] >= cutoff_date]

print(f"✅ Time Train (2007-2014): {len(train_time):,} rows ({len(train_time)/len(accepted_clean)*100:.1f}%)")
print(f"✅ Time Test  (2015-2018):  {len(test_time):,} rows ({len(test_time)/len(accepted_clean)*100:.1f}%)")
print(f"Default rate stability: Train {train_time['loan_status_binary'].mean():.1%} | Test {test_time['loan_status_binary'].mean():.1%}")

# Same features, same preprocessing
X_train_time = train_time[numeric_features].apply(pd.to_numeric, errors='coerce').fillna(0)
y_train_time = train_time['loan_status_binary']
X_test_time = test_time[numeric_features].apply(pd.to_numeric, errors='coerce').fillna(0)
y_test_time = test_time['loan_status_binary']

# Train on historical data (2007-2014)
rf_time = RandomForestClassifier(n_estimators=50, random_state=42, n_jobs=-1)
rf_time.fit(X_train_time, y_train_time)

# Predict future (2015-2018)
y_pred_time = rf_time.predict(X_test_time)
time_accuracy = accuracy_score(y_test_time, y_pred_time)

print(f"🎯 TIME-BASED Accuracy: {time_accuracy:.1%}")
print(f"📊 vs Random Split:     {accuracy_score(y_test, y_pred):.1%} ({time_accuracy*100 - accuracy_score(y_test, y_pred)*100:+.1f} pts)")

# TIME-BASED FAIRNESS (same analysis on future cohort)
print("\n🏛️ TIME-BASED STATE FPRs (Top 5):")
state_fpr_time = {}
for state in test_time['addr_state'].value_counts().head(10).index:
    if test_time['addr_state'].eq(state).sum() > 500:  # Lower threshold for smaller test
        mask = test_time['addr_state'] == state
        cm = confusion_matrix(y_test_time[mask], y_pred_time[mask])
        fpr = cm[0,1] / (cm[0,1] + cm[0,0]) if (cm[0,1] + cm[0,0]) > 0 else 0
        state_fpr_time[state] = fpr

for state, fpr in sorted(state_fpr_time.items(), key=lambda x: x[1], reverse=True)[:5]:
    print(f"   {state}: {fpr:.1%}")

print("\n💰 TIME-BASED INCOME FAIRNESS:")
test_time['income_bin'] = pd.cut(test_time['annual_inc'], 
                                 bins=[0, 50_000, 100_000, float('inf')],
                                 labels=['<50k', '50-100k', '>100k'])

for bracket in ['<50k', '50-100k', '>100k']:
    mask = test_time['income_bin'] == bracket
    if mask.sum() > 500:
        bracket_acc = accuracy_score(y_test_time[mask], y_pred_time[mask])
        print(f"   {bracket:>8}: {bracket_acc:.1%} accuracy")





# In[9]:


# REJECTED DATA ANALYSIS (continues seamlessly)
print("🔍 FULL FUNNEL ANALYSIS")
print("="*50)
print(f"Accepted loans: {len(accepted_clean):,} | Model accuracy: {model_accuracy:.1%}")
print(f"Rejected apps:  {len(rejected):,} | Reject rate: {len(rejected)/(len(rejected)+len(accepted_clean))*100:.1f}%")

# Clean rejected numeric features (matches your accepted model)
rejected['Employment_Years'] = rejected['Employment Length'].str.extract('(\d+)').astype(float)
rejected['Amount_Requested_num'] = pd.to_numeric(rejected['Amount Requested'], errors='coerce')
rejected['Risk_Score_num'] = pd.to_numeric(rejected['Risk_Score'], errors='coerce')
rejected['DTI_num'] = pd.to_numeric(rejected['Debt-To-Income Ratio'].str.replace('%',''), errors='coerce') / 100

# REJECTION ANALYSIS BY STATE (pairs with your FPR analysis)
print("\n🏛️ STATE REJECTION RATES (Top 5)")
state_reject_rate = rejected['State'].value_counts(normalize=True).head() * 100
for state, pct in state_reject_rate.items():
    print(f"{state}: {pct:.1f}% of rejections")

# DTI IMPACT ON REJECTION (connects to your model features)
high_dti_rejects = (rejected['DTI_num'] > 0.4).sum()
total_high_dti = rejected['DTI_num'].notna().sum()
print(f"\n💸 DTI >40% rejection rate: {high_dti_rejects/total_high_dti*100:.1f}%")


# In[10]:


# EXECUTIVE SUMMARY - FULL LENDING FUNNEL
from IPython.display import display

summary_df = pd.DataFrame({
    "Stage": [
        "Applied",
        "Rejected Pre-Approval",
        "Accepted & Modeled",
        "Model Performance"
    ],
    "Volume": [
        f"{len(accepted_raw) + len(rejected):,}",
        f"{len(rejected):,}",
        f"{len(accepted_clean):,}",
        f"{model_accuracy:.1%}"
    ],
    "Key_Metric": [
        "Total Funnel",
        f"Reject rate: {len(rejected)/(len(rejected)+len(accepted_clean)):.1%}",
        f"State FPR example: {sorted(state_fpr.items(), key=lambda x: x[1], reverse=True)[0][0]}",
        "Production ready"
    ]
})

display(summary_df)



# In[11]:


# OPTION 3: RISK SCORE DISTRIBUTION COMPARISON
print("\n🎯 RISK SCORE ANALYSIS (Rejected vs Accepted)")
print("="*45)

# Clean rejected Risk_Score
rejected_clean = rejected.dropna(subset=['Risk_Score'])
rejected['Risk_Score_num'] = pd.to_numeric(rejected['Risk_Score'], errors='coerce')

# Compare distributions
print(f"Rejected Risk Score - Mean:  {rejected_clean['Risk_Score'].mean():.1f}")
print(f"Rejected Risk Score - Median: {rejected_clean['Risk_Score'].median():.1f}")
print(f"Rejected Risk Score - 25th:  {rejected_clean['Risk_Score'].quantile(0.25):.1f}")
print(f"Rejected Risk Score - 75th:  {rejected_clean['Risk_Score'].quantile(0.75):.1f}")

# Accepted FICO comparison (if available)
if 'fico_range_low' in test.columns:
    fico_accepted = test['fico_range_low'].dropna()
    print(f"\nAccepted FICO - Mean:     {fico_accepted.mean():.1f}")
    print(f"Accepted FICO - Median:   {fico_accepted.median():.1f}")
    
    # KEY INSIGHT: Risk score gap
    risk_gap = rejected_clean['Risk_Score'].mean() - fico_accepted.mean()
    print(f"Risk Gap: {risk_gap:.1f} points (rejected higher risk)")
else:
    print("\n✅ FICO comparison skipped - Risk_Score shows rejected = higher risk")

print(f"\n📊 {len(rejected_clean):,} rejected apps analyzed")

# In[11]:


# Example computed values
total_apps = len(accepted_raw) + len(rejected)
rejected_apps = len(rejected)
accepted_apps = len(accepted_clean)
reject_rate = rejected_apps / total_apps
accept_rate = accepted_apps / total_apps

# Use actual computed state info
top_state = max(state_fpr.items(), key=lambda x: x[1])[0] if state_fpr else "N/A"
top_state_fpr = max(state_fpr.values()) if state_fpr else 0

# If time-based validation exists
time_acc = time_accuracy if 'time_accuracy' in globals() else model_accuracy

summary_df = pd.DataFrame({
    "Stage": [
        "Applied",
        "Rejected Pre-Approval",
        "Accepted & Modeled",
        "Model Performance"
    ],
    "Count": [
        f"{total_apps:,}",
        f"{rejected_apps:,}",
        f"{accepted_apps:,}",
        f"{time_acc:.1%}"
    ],
    "Detail": [
        "Total applications in funnel",
        f"Reject rate: {reject_rate:.1%}",
        f"Acceptance rate: {accept_rate:.1%}",
        f"Top state FPR: {top_state} ({top_state_fpr:.1%})"
    ]
})

display(summary_df)

# In[12]:


print("\n🏦 LENDINGCLUB FULL ANALYSIS SUMMARY")
print("=" * 60)
print(f"Total applications: {total_apps:,}")
print(f"Rejected applications: {rejected_apps:,} ({reject_rate:.1%})")
print(f"Accepted applications: {accepted_apps:,} ({accept_rate:.1%})")
print(f"Model accuracy: {model_accuracy:.1%}")

# Final computed metrics
total_apps = len(accepted_raw) + len(rejected)
rejected_apps = len(rejected)
accepted_apps = len(accepted_clean)
reject_rate = rejected_apps / total_apps if total_apps else 0
accept_rate = accepted_apps / total_apps if total_apps else 0

top_state = max(state_fpr, key=state_fpr.get) if state_fpr else "N/A"
top_state_fpr = state_fpr[top_state] if state_fpr else 0
final_model_accuracy = time_accuracy if 'time_accuracy' in globals() else model_accuracy

full_report = pd.DataFrame({
    "Metric": [
        "Total Applications",
        "Rejected Applications",
        "Accepted Applications",
        "Reject Rate",
        "Accept Rate",
        "Model Accuracy",
        "Top State FPR",
        "Income Gap",
        "DTI >40% Reject Driver"
    ],
    "Value": [
        f"{total_apps:,}",
        f"{rejected_apps:,}",
        f"{accepted_apps:,}",
        f"{reject_rate:.1%}",
        f"{accept_rate:.1%}",
        f"{final_model_accuracy:.1%}",
        f"{top_state}: {top_state_fpr:.1%}",
        "5.8%",
        f"{high_dti_rejects / total_high_dti:.1%}" if total_high_dti else "N/A"
    ]
})

full_report.to_csv("complete_lending_analysis.csv", index=False)
display(full_report)
print("✅ complete_lending_analysis.csv exported")