Further isolate environment variables to fix inconsistent results

common/common_constants.py

@@ -2,5 +2,4 @@
 EMAIL = "admin@causalbench.org"
 REPLY_TO_ADDRESS = "contact@causalbench.org"
 EMAIL_PASSWORD = ""
-TEMP_DIR = "/tmp"
 RANDOM_SEED = 42

common/yaml_to_csv.py

@@ -6,12 +6,7 @@
 from rapidfuzz import process, fuzz
 from causalbench.modules import Dataset
 from causalbench.modules import Run
-from causalbench.modules.context import Context
 
-from common.common_constants import TEMP_DIR
-
-
-os.environ["MPLCONFIGDIR"] = os.path.join(TEMP_DIR, "mplconfig")
 
 # Set working directory to parent dir
 # parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))

helper_services/causal_analysis_helper.py

@@ -1,21 +1,15 @@
 from collections import defaultdict
-import shutil
 import os
-import tempfile
 from urllib.parse import urlparse
 import pandas as pd
 import numpy as np
-import requests
 import networkx as nx
 from dowhy import CausalModel
-from common.common_constants import RANDOM_SEED, TEMP_DIR
+from common.common_constants import RANDOM_SEED
 from common.yaml_to_csv import main as process_yaml_data, headers
 from sklearn.preprocessing import LabelEncoder, StandardScaler
 
 
-os.environ["MPLCONFIGDIR"] = os.path.join(TEMP_DIR, "mplconfig")
-
-
 def compute_CATE(data, treatment, outcome, graph):
     try:
         data_clean = data.copy()

helper_services/download_helper.py

@@ -24,7 +24,7 @@ def download_zip_from_url(url, download_dir):
             for chunk in response.iter_content(chunk_size=8192):
                 f.write(chunk)
 
-        print(f"Downloaded: {filename}")
+        print(f"Downloaded: {filepath}")
         return filepath
 
     except Exception as e:

helper_services/report_helper.py

@@ -1,4 +1,5 @@
 import os
+import tempfile
 import numpy as np
 from openpyxl.styles import Font
 from openpyxl.utils import get_column_letter
@@ -13,8 +14,6 @@
 
 import yaml
 
-from common.common_constants import TEMP_DIR
-
 
 def generate_report(outcome_column, causal_analysis_results, unique_id, run_ids, filters):
     # Set up parameters
@@ -23,7 +22,7 @@ def generate_report(outcome_column, causal_analysis_results, unique_id, run_ids,
 
     # Create a YAML file
     yaml_filename = f"causal_analysis_results_{timestamp}-{unique_id}.yaml"
-    yaml_filepath = os.path.join(TEMP_DIR, yaml_filename)
+    yaml_filepath = os.path.join(tempfile.gettempdir(), yaml_filename)
 
     with open(yaml_filepath, 'w') as yaml_file:
         yaml.dump(causal_analysis_results, yaml_file, default_flow_style=False, indent=4, sort_keys=False)
@@ -32,7 +31,7 @@ def generate_report(outcome_column, causal_analysis_results, unique_id, run_ids,
 
     # Create a PDF document
     pdf_filename = f"causal_explanation_report_{timestamp}-{unique_id}.pdf"
-    pdf_filepath = os.path.join(TEMP_DIR, pdf_filename)
+    pdf_filepath = os.path.join(tempfile.gettempdir(), pdf_filename)
     doc = SimpleDocTemplate(
         pdf_filepath,
         pagesize=LETTER,
@@ -44,7 +43,7 @@ def generate_report(outcome_column, causal_analysis_results, unique_id, run_ids,
 
     # Create an Excel file
     xlsx_filename = f"causal_recommendations_{timestamp}-{unique_id}.xlsx"
-    xlsx_filepath = os.path.join(TEMP_DIR, xlsx_filename)
+    xlsx_filepath = os.path.join(tempfile.gettempdir(), xlsx_filename)
 
     # Colors
     tab_h_bg_col = colors.HexColor("#95979d")    # table header background color

lambda_function.py

@@ -1,6 +1,6 @@
-import atexit
 from collections import defaultdict
 import os
+import tempfile
 
 import causalbench
 from helper_services.causal_analysis_helper import run_causal_analysis
@@ -12,7 +12,6 @@
 from helper_services.hp_dtype_helper import get_hp_dtypes
 from helper_services.mail_helper import send_email
 import numpy as np
-from common.common_constants import TEMP_DIR
 
 
 def build_email_body(causal_analysis_results, event):
@@ -64,12 +63,30 @@ def build_email_body(causal_analysis_results, event):
     return "\n".join(lines)
 
 
+def configure_env():
+    """
+    Directory setup to ensure isolation
+    """
+    # fake temporary directory
+    temp_dir = tempfile.mkdtemp()
+    tempfile.tempdir = None
+    os.environ["TMPDIR"] = temp_dir
+    os.environ["TEMP"] = temp_dir
+    os.environ["TMP"] = temp_dir
+
+    # fake home directory
+    home_dir = os.path.join(temp_dir, "home")
+    os.makedirs(home_dir, exist_ok=True)
+    os.environ["HOME"] = home_dir
+    os.environ["USERPROFILE"] = home_dir
+
+    # fake mpl config directory
+    os.environ["MPLCONFIGDIR"] = os.path.join(temp_dir, "mplconfig")
+
+
 def handler(event, context):
-    # create fake home to ensure isolation
-    fake_home = os.path.abspath(os.path.join(TEMP_DIR, "home"))
-    os.makedirs(fake_home, exist_ok=True)
-    os.environ["HOME"] = fake_home
-    os.environ["USERPROFILE"] = fake_home
+    # configure the environment variables
+    configure_env()
 
     # set JWT token
     causalbench.services.auth.__access_token = event.get('jwt_token', None)
@@ -112,8 +129,7 @@ def handler(event, context):
         try:
             if len(dimensions) > 0:
                 cols = ["HP." + dim for dim in dimensions.keys()]
-                # data = list(group_data["data"][cols].itertuples(index=False, name=None))
-                # group_data['recommendations'] = run_causal_recommendation(data, dimensions, hp_dtypes, max_points)
+
                 sample_frame = group_data["data"][cols + ["outcome"]].copy()
                 group_data['recommendations'] = run_g2s_causal_recommendation(sample_frame, dimensions, hp_dtypes, max_points)
             else:
@@ -136,11 +152,6 @@ def handler(event, context):
     except Exception as e:
         print(f"Error sending email: {e}")
 
-    # # remove the attachments after sending the email
-    # for attachment in attachments:
-    #     if os.path.exists(attachment):
-    #         atexit.register(lambda path=attachment: os.remove(path))
-
     response = {
         "analysis_results": causal_analysis_results
     }