diff --git a/.gitignore b/.gitignore index dbcbf50..e927a10 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ __pycache__/ tmp/ common/common_constants.py home/ +.claude/* diff --git a/common/common_constants.py b/common/common_constants.py index 04da309..df778f9 100644 --- a/common/common_constants.py +++ b/common/common_constants.py @@ -1,5 +1,4 @@ CAUSAL_ANALYSIS_FAILED = "Causal analysis failed. Please try again later." -CAUSAL_ANALYSIS_EMAIL_BODY = "Please find the causal analysis results in the pdf document attached to this email." EMAIL = "admin@causalbench.org" REPLY_TO_ADDRESS = "contact@causalbench.org" EMAIL_PASSWORD = "" diff --git a/helper_services/causal_analysis_helper.py b/helper_services/causal_analysis_helper.py index 9c96107..441b554 100644 --- a/helper_services/causal_analysis_helper.py +++ b/helper_services/causal_analysis_helper.py @@ -116,7 +116,11 @@ def run_causal_analysis(download_dir, } encode = [] - try: + raw_df = pd.DataFrame() + hyperparameters = [] + experiment_count = 0 + load_error = None + try: if download_dir: print(f"Processing ZIP files from {download_dir}") raw_df = process_yaml_data(download_dir, headers) @@ -164,6 +168,7 @@ def run_causal_analysis(download_dir, except Exception as e: print(f"Error loading data: {e}") + load_error = str(e) if group_by_metric: df_columns = ['dataset', 'model', 'metric'] + hyperparameters + ['outcome'] @@ -206,7 +211,8 @@ def run_causal_analysis(download_dir, print(label_encoder.classes_) df = df.dropna() - print(f"After cleaning: {len(df)} experiments remain") + experiment_count = len(df) + print(f"After cleaning: {experiment_count} experiments remain") df = df.sort_values(['dataset'] + [col for col in sorted(df.columns) if col != 'dataset']).reset_index(drop=True) @@ -286,4 +292,36 @@ def run_causal_analysis(download_dir, except Exception as e: print(f"Error in causal analysis: {e}") + insufficient_data = False + insufficient_data_reason = None + + if load_error is not None: + insufficient_data = True + insufficient_data_reason = f"Error loading data: {load_error}" + elif raw_df.empty: + insufficient_data = True + insufficient_data_reason = "No data files could be downloaded from provided URLs" + elif len(hyperparameters) == 0: + insufficient_data = True + insufficient_data_reason = "No hyperparameters with sufficient variation found" + elif experiment_count < 2: + insufficient_data = True + insufficient_data_reason = "Too few experiments after data cleaning (need ≥ 2)" + elif group_results: + all_nan = all( + all(np.isnan(v) for v in group_data.get("effects", {}).values()) + for group_data in group_results.values() + if "effects" in group_data + ) + if all_nan: + insufficient_data = True + insufficient_data_reason = "Causal effects could not be estimated (insufficient variation in data)" + + group_results["_metadata"] = { + "experiment_count": experiment_count, + "hyperparameter_count": len(hyperparameters), + "insufficient_data": insufficient_data, + "insufficient_data_reason": insufficient_data_reason, + } + return group_results, download_dir diff --git a/helper_services/report_helper.py b/helper_services/report_helper.py index 36f7114..5f51260 100644 --- a/helper_services/report_helper.py +++ b/helper_services/report_helper.py @@ -169,14 +169,16 @@ def generate_report(outcome_column, causal_analysis_results, unique_id, run_ids, elements.append(separator) - if len(causal_analysis_results) == 0: + analysis_groups = {k: v for k, v in causal_analysis_results.items() if k != "_metadata"} + + if len(analysis_groups) == 0: elements.append(Paragraph(f"Analysis: Effects on {outcome_column} (0 experiments)", header_style)) elements.append(Paragraph("Insufficient data to perform causal analysis.", body_style)) elements.append(separator) - + else: # Process data - for group, group_data in causal_analysis_results.items(): + for group, group_data in analysis_groups.items(): table_data = [["Variable", "Effect", "Strength"]] if group_data['experiments'] == 1: diff --git a/lambda_function.py b/lambda_function.py index 588744d..33068ca 100644 --- a/lambda_function.py +++ b/lambda_function.py @@ -12,7 +12,56 @@ from helper_services.hp_dtype_helper import get_hp_dtypes from helper_services.mail_helper import send_email import numpy as np -from common.common_constants import CAUSAL_ANALYSIS_EMAIL_BODY, TEMP_DIR +from common.common_constants import TEMP_DIR + + +def build_email_body(causal_analysis_results, event): + outcome_column = event.get('outcome_column', 'Time.Duration') + filters = event.get('filters', None) + metadata = causal_analysis_results.get('_metadata', {}) + + experiment_count = metadata.get('experiment_count', 0) + insufficient_data = metadata.get('insufficient_data', False) + insufficient_data_reason = metadata.get('insufficient_data_reason', None) + + lines = ["CausalBench+ Causal Analysis Report", ""] + lines.append(f"Outcome metric: {outcome_column}") + lines.append(f"Experiments: Effects on {outcome_column} ({experiment_count} experiments)") + + if filters: + filter_str = ", ".join(f"{k}={v}" for k, v in filters.items()) if isinstance(filters, dict) else str(filters) + lines.append(f"Filters applied: {filter_str}") + + lines.append("") + + if insufficient_data: + lines.append("INSUFFICIENT DATA: Causal effects could not be computed.") + if insufficient_data_reason: + lines.append(f"Reason: {insufficient_data_reason}") + lines.append("") + lines.append("To get results, run more experiments with varied hyperparameter configurations.") + lines.append("Minimum requirements: ≥ 2 data points per variable, ≥ 2 unique values per hyperparameter.") + else: + all_effects = {} + for group, group_data in causal_analysis_results.items(): + if group == "_metadata": + continue + for k, v in group_data.get("effects", {}).items(): + if isinstance(v, (int, float)) and math.isfinite(v): + all_effects[k] = v + + if all_effects: + sorted_effects = sorted(all_effects.items(), key=lambda x: abs(x[1]), reverse=True)[:3] + lines.append("Top causal effects:") + for hp, effect in sorted_effects: + hp_name = hp.split(".", 1)[1] if "." in hp else hp + sign = "+" if effect >= 0 else "" + lines.append(f" {hp_name}: {sign}{effect:.4f}") + + lines.append("") + lines.append("Full results are in the attached PDF report.") + + return "\n".join(lines) def handler(event, context): @@ -47,11 +96,13 @@ def handler(event, context): # find all causal recommendations for group, group_data in causal_analysis_results.items(): + if group == "_metadata": + continue effects = group_data["effects"] dimensions = defaultdict(dict) for k, v in effects.items(): k = k.split(".")[1] # Remove 'HP.' prefix - if k in list(event.get('hyperparameter_limits', {}).keys()) and v != 0: + if k in list(event.get('hyperparameter_limits', {}).keys()) and math.isfinite(v) and v != 0: dimensions[k]['strength'] = v dimensions[k]['min_val'] = event.get('hyperparameter_limits', {})[k]['min'] dimensions[k]['max_val'] = event.get('hyperparameter_limits', {})[k]['max'] @@ -81,7 +132,7 @@ def handler(event, context): attachments.append(xlsx_filepath) try: - send_email(event.get('user_email'), "[CausalBench] Causal Analysis Results", CAUSAL_ANALYSIS_EMAIL_BODY, attachments=attachments) + send_email(event.get('user_email'), "[CausalBench] Causal Analysis Results", build_email_body(causal_analysis_results, event), attachments=attachments) except Exception as e: print(f"Error sending email: {e}")