perform_statistical_analysis.py

import os
import pandas as pd
import pymannkendall as mk
from esda.moran import Moran
from libpysal.weights import lat2W

# Create a directory for statistical analysis
statistical_analysis_dir = 'statistical_analysis'
if not os.path.exists(statistical_analysis_dir):
    os.makedirs(statistical_analysis_dir)

# Load combined data
combined_data = pd.read_csv('combined_trends_data.csv', index_col=0, parse_dates=True)

# Mann-Kendall Test
def mann_kendall_test():
    results = {}
    for keyword in combined_data.columns:
        if keyword == 'City':
            continue
        trend_result = mk.original_test(combined_data[keyword].dropna())
        results[keyword] = trend_result
        print(f"Mann-Kendall Test for {keyword}: Trend = {trend_result.trend}, p-value = {trend_result.p}")
    return results

# Moran's I Calculation
def morans_i_test():
    results = {}
    for keyword in combined_data.columns:
        if keyword == 'City':
            continue
        # Create a spatial weights matrix for 4 cities
        w = lat2W(2, 2)  # Adjust if the number of cities changes
        # Extract data for Moran's I calculation
        city_means = combined_data.groupby('City')[keyword].mean()
        moran = Moran(city_means, w)
        results[keyword] = moran
        print(f"Moran's I for {keyword}: I = {moran.I}, p-value = {moran.p_sim}")
    return results

# Perform all statistical analyses
mk_results = mann_kendall_test()
mi_results = morans_i_test()

# Save results to files with UTF-8 encoding
with open(f'{statistical_analysis_dir}/mann_kendall_results.txt', 'w', encoding='utf-8') as f:
    for keyword, result in mk_results.items():
        f.write(f"{keyword}: Trend = {result.trend}, p-value = {result.p}\n")

with open(f'{statistical_analysis_dir}/morans_i_results.txt', 'w', encoding='utf-8') as f:
    for keyword, moran in mi_results.items():
        f.write(f"{keyword}: I = {moran.I}, p-value = {moran.p_sim}\n")

print("Statistical analysis complete. Check the 'statistical_analysis' directory for results.")