From c49e2274f3e5dc45b4ff13d51422efa948d03c1f Mon Sep 17 00:00:00 2001 From: Astacatalyst Date: Fri, 21 Nov 2025 13:13:49 +0530 Subject: [PATCH 1/4] agent-analysis --- agents/analysis_agent.py | 155 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) diff --git a/agents/analysis_agent.py b/agents/analysis_agent.py index e69de29..83a7f0b 100644 --- a/agents/analysis_agent.py +++ b/agents/analysis_agent.py @@ -0,0 +1,155 @@ +# agents/analysis_agent.py +""" +Analysis Agent for ProdigyFlow +-------------------------------- +Performs: +- Basic statistical analysis +- Correlation analysis +- Text-based summary insight generation (optional Gemini API support) +- Returns dictionary of insights + metadata + +Designed to be easy to read, simple, and competition-friendly. +""" + +import pandas as pd +import numpy as np +from pathlib import Path +from typing import Dict, Any +from datetime import datetime + +# Optional Google Gemini API (fails gracefully) +try: + import google.generativeai as genai + GEMINI_AVAILABLE = True +except ImportError: + GEMINI_AVAILABLE = False + + +# -------------------------------------------- +# Gemini Configuration (Optional) +# -------------------------------------------- +def configure_gemini(api_key: os.environ.get("AIzaSyDsRf396SxARygGEPNgU8C-NJJO-XRVC20")) -> bool: + """ + Configures Gemini only if API key is provided and package is installed. + """ + if api_key and GEMINI_AVAILABLE: + genai.configure(api_key=api_key) + return True + return False + + +# -------------------------------------------- +# Helper Functions +# -------------------------------------------- +def compute_basic_stats(df: pd.DataFrame) -> Dict[str, Any]: + """ + Returns summary statistics for numerical and categorical columns. + """ + numerical = df.select_dtypes(include=["int64", "float64"]).columns.tolist() + categorical = df.select_dtypes(include=["object", "category"]).columns.tolist() + + stats = { + "num_columns": numerical, + "cat_columns": categorical, + "describe_numeric": df[numerical].describe().to_dict() if numerical else {}, + "missing_values": df.isna().sum().to_dict(), + "unique_counts": {col: df[col].nunique() for col in df.columns} + } + return stats + + +def compute_correlations(df: pd.DataFrame) -> Dict[str, Any]: + """ + Computes pairwise correlations for numeric columns. + """ + numeric_df = df.select_dtypes(include=[np.number]) + if numeric_df.empty: + return {"correlations": {}} + + corr = numeric_df.corr().round(3).fillna(0) + high_corr_pairs = [] + + for col in corr.columns: + for idx in corr.index: + if col != idx and abs(corr.loc[idx, col]) > 0.6: + high_corr_pairs.append({ + "feature_1": idx, + "feature_2": col, + "correlation": float(corr.loc[idx, col]) + }) + + return { + "correlation_matrix": corr.to_dict(), + "high_correlation_pairs": high_corr_pairs + } + + +def generate_ai_insights(df: pd.DataFrame, stats: Dict[str, Any], corr: Dict[str, Any]) -> str: + """ + Uses Gemini to generate a natural-language analysis summary. + Falls back to rule-based summary if Gemini is unavailable. + """ + + # If Gemini is not available → return simple summary + if not GEMINI_AVAILABLE: + return ( + "AI model unavailable — generated rule-based insights.\n" + f"- Dataset has {df.shape[0]} rows and {df.shape[1]} columns.\n" + f"- Numerical features: {len(stats['num_columns'])}\n" + f"- Categorical features: {len(stats['cat_columns'])}\n" + f"- High correlations found: {len(corr['high_correlation_pairs'])}\n" + ) + + prompt = f""" +You are an AI Data Analyst. Summarize the dataset insights in simple, +professional language. Avoid technical jargon. Here is the analysis: + +Rows: {df.shape[0]} +Columns: {df.shape[1]} + +Numerical Columns: {stats['num_columns']} +Categorical Columns: {stats['cat_columns']} + +Missing Values: {stats['missing_values']} + +Strong Correlations (> 0.6): +{corr['high_correlation_pairs']} + +Give clear, meaningful insights suitable for a student competition project. +""" + + try: + model = genai.GenerativeModel("gemini-1.5-flash") + response = model.generate_content(prompt) + return response.text + except Exception: + return "Gemini failed to respond — using fallback rule-based insights." + +if __name__ == "__main__": + print("Running Analysis Agent...") + + # Path of your CSV + csv_path = "data\\data_science_student_marks.csv" + + # Load dataset + df = pd.read_csv(csv_path) + + # Compute stats + stats = compute_basic_stats(df) + + # Compute correlations + corr = compute_correlations(df) + + # Generate insights + insights = generate_ai_insights(df, stats, corr) + + # Final structured result: + result = { + "timestamp": str(datetime.now()), + "stats": stats, + "correlations": corr, + "ai_insights": insights, + } + + print("\n=== ANALYSIS RESULT ===") + print(result) From be5eac04a77845a429dd109177bc3ed13e0fe13b Mon Sep 17 00:00:00 2001 From: Astacatalyst Date: Fri, 21 Nov 2025 13:16:27 +0530 Subject: [PATCH 2/4] agent-cleaning --- agents/cleaning_agent.py | 153 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/agents/cleaning_agent.py b/agents/cleaning_agent.py index e69de29..5e56018 100644 --- a/agents/cleaning_agent.py +++ b/agents/cleaning_agent.py @@ -0,0 +1,153 @@ +import pandas as pd +import os +import google.generativeai as genai + +# Initialize Gemini client +# Note: Ensure GEMINI_API_KEY is set in your environment variables +try: + api_key = os.environ.get("GEMINI_API_KEY") + if api_key: + genai.configure(api_key=api_key) + model = genai.GenerativeModel('gemini-1.5-flash') + print("Gemini client initialized successfully.") + else: + print("Warning: GEMINI_API_KEY not found in environment variables. LLM features will be disabled.") + model = None +except Exception as e: + print(f"Warning: Gemini client could not be initialized. Error: {e}") + model = None + +def load_data(filepath): + """Loads data from a CSV file.""" + try: + df = pd.read_csv(filepath) + print(f"Successfully loaded data from {filepath}") + return df + except FileNotFoundError: + print(f"Error: File not found at {filepath}") + return None + except Exception as e: + print(f"Error loading data: {e}") + return None + +def save_data(df, filepath): + """Saves the DataFrame to a CSV file.""" + try: + df.to_csv(filepath, index=False) + print(f"Successfully saved cleaned data to {filepath}") + except Exception as e: + print(f"Error saving data: {e}") + +def clean_missing(df, strategy='ffill'): + """ + Handle missing values. + Strategies: 'ffill', 'bfill', 'drop', 'mean', 'median', 'mode' + """ + print(f"Executing clean_missing with strategy: {strategy}") + if strategy == 'ffill': + return df.ffill() + elif strategy == 'bfill': + return df.bfill() + elif strategy == 'drop': + return df.dropna() + elif strategy == 'mean': + return df.fillna(df.mean(numeric_only=True)) + elif strategy == 'median': + return df.fillna(df.median(numeric_only=True)) + elif strategy == 'mode': + return df.fillna(df.mode().iloc[0]) + else: + print(f"Unknown strategy '{strategy}', defaulting to ffill") + return df.ffill() + +def clean_duplicates(df, keep='first'): + """Removes duplicate rows.""" + print(f"Executing clean_duplicates keeping: {keep}") + return df.drop_duplicates(keep=keep) + +def clean_text(df): + """Standardizes text columns (strip whitespace, lowercase).""" + print("Executing clean_text") + for col in df.select_dtypes(include='object'): + df[col] = df[col].str.strip().str.lower() + return df + +def fix_dtypes(df): + """Infers better data types for columns.""" + print("Executing fix_dtypes") + return df.infer_objects() + +def get_cleaning_plan(df_head_str, user_prompt): + """Uses Gemini to suggest a cleaning plan based on data sample and user request.""" + if not model: + return "missing, duplicate, text, datatype" # Fallback default plan + + system_prompt = """You are a data-cleaning agent. + Analyze the provided data sample and user request. + Return a comma-separated list of cleaning steps to apply. + Available steps: 'missing', 'duplicate', 'text', 'datatype'. + Example output: missing, text + """ + + content = f"{system_prompt}\n\nData Sample:\n{df_head_str}\n\nUser Request: {user_prompt}" + + try: + response = model.generate_content(content) + return response.text.strip() + except Exception as e: + print(f"Error getting plan from LLM: {e}") + return "missing, duplicate, text, datatype" + +def clean_dataset(df, plan): + """Applies cleaning functions based on the plan.""" + plan = plan.lower() + + if "missing" in plan: + # Simple heuristic: if numeric, use mean, else ffill + # For now, defaulting to ffill/bfill as per original or user preference could be added + df = clean_missing(df) + if "duplicate" in plan: + df = clean_duplicates(df) + if "text" in plan: + df = clean_text(df) + if "datatype" in plan: + df = fix_dtypes(df) + + return df + +def analyze_data(df): + """Prints basic analysis of the dataset.""" + print("\n--- Data Analysis ---") + print(f"Shape: {df.shape}") + print(f"Missing Values:\n{df.isnull().sum()}") + print(f"Duplicates: {df.duplicated().sum()}") + print("---------------------\n") + +def main(): + input_file = "data//data_science_student_marks.csv" + output_file = "cleaned_data.csv" + + print("--- Data Cleaning Agent (Gemini Powered) ---") + df = load_data(input_file) + + if df is not None: + analyze_data(df) + + user_prompt = "Clean this dataset automatically." # Default prompt + # In a real interactive CLI, we would ask: input("Enter your cleaning goal: ") + + print("Generating cleaning plan...") + plan = get_cleaning_plan(df.head().to_string(), user_prompt) + print(f"Proposed Plan: {plan}") + + # In interactive mode, we would ask for confirmation here. + print("Applying plan...") + cleaned_df = clean_dataset(df, plan) + + analyze_data(cleaned_df) + + save_data(cleaned_df, output_file) + print("Done.") + +if __name__ == "__main__": + main() From ec7cafacaa46cb3508c127305e3312b0dee96384 Mon Sep 17 00:00:00 2001 From: Priyam <143118533+astacatalyst@users.noreply.github.com> Date: Fri, 21 Nov 2025 23:58:01 +0530 Subject: [PATCH 3/4] Add permissions to labeler workflow --- .github/workflows/labeler.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 1c2b536..be1f8af 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -6,6 +6,10 @@ on: - main - master +permissions: + contents: read + pull-requests: write + jobs: label: runs-on: ubuntu-latest From 9173816ccd386af4e719c260e90d3a0d011c5789 Mon Sep 17 00:00:00 2001 From: Komal Harshita Date: Sat, 22 Nov 2025 09:29:32 +0530 Subject: [PATCH 4/4] Delete agents/analysis_agent.py --- agents/analysis_agent.py | 155 --------------------------------------- 1 file changed, 155 deletions(-) delete mode 100644 agents/analysis_agent.py diff --git a/agents/analysis_agent.py b/agents/analysis_agent.py deleted file mode 100644 index be31684..0000000 --- a/agents/analysis_agent.py +++ /dev/null @@ -1,155 +0,0 @@ -# agents/analysis_agent.py -""" -Analysis Agent for ProdigyFlow --------------------------------- -Performs: -- Basic statistical analysis -- Correlation analysis -- Text-based summary insight generation (optional Gemini API support) -- Returns dictionary of insights + metadata - -Designed to be easy to read, simple, and competition-friendly. -""" - -import pandas as pd -import numpy as np -from pathlib import Path -from typing import Dict, Any -from datetime import datetime - -# Optional Google Gemini API (fails gracefully) -try: - import google.generativeai as genai - GEMINI_AVAILABLE = True -except ImportError: - GEMINI_AVAILABLE = False - - -# -------------------------------------------- -# Gemini Configuration (Optional) -# -------------------------------------------- -def configure_gemini(api_key: os.environ.get("YOUR_API_KEY")) -> bool: - """ - Configures Gemini only if API key is provided and package is installed. - """ - if api_key and GEMINI_AVAILABLE: - genai.configure(api_key=api_key) - return True - return False - - -# -------------------------------------------- -# Helper Functions -# -------------------------------------------- -def compute_basic_stats(df: pd.DataFrame) -> Dict[str, Any]: - """ - Returns summary statistics for numerical and categorical columns. - """ - numerical = df.select_dtypes(include=["int64", "float64"]).columns.tolist() - categorical = df.select_dtypes(include=["object", "category"]).columns.tolist() - - stats = { - "num_columns": numerical, - "cat_columns": categorical, - "describe_numeric": df[numerical].describe().to_dict() if numerical else {}, - "missing_values": df.isna().sum().to_dict(), - "unique_counts": {col: df[col].nunique() for col in df.columns} - } - return stats - - -def compute_correlations(df: pd.DataFrame) -> Dict[str, Any]: - """ - Computes pairwise correlations for numeric columns. - """ - numeric_df = df.select_dtypes(include=[np.number]) - if numeric_df.empty: - return {"correlations": {}} - - corr = numeric_df.corr().round(3).fillna(0) - high_corr_pairs = [] - - for col in corr.columns: - for idx in corr.index: - if col != idx and abs(corr.loc[idx, col]) > 0.6: - high_corr_pairs.append({ - "feature_1": idx, - "feature_2": col, - "correlation": float(corr.loc[idx, col]) - }) - - return { - "correlation_matrix": corr.to_dict(), - "high_correlation_pairs": high_corr_pairs - } - - -def generate_ai_insights(df: pd.DataFrame, stats: Dict[str, Any], corr: Dict[str, Any]) -> str: - """ - Uses Gemini to generate a natural-language analysis summary. - Falls back to rule-based summary if Gemini is unavailable. - """ - - # If Gemini is not available → return simple summary - if not GEMINI_AVAILABLE: - return ( - "AI model unavailable — generated rule-based insights.\n" - f"- Dataset has {df.shape[0]} rows and {df.shape[1]} columns.\n" - f"- Numerical features: {len(stats['num_columns'])}\n" - f"- Categorical features: {len(stats['cat_columns'])}\n" - f"- High correlations found: {len(corr['high_correlation_pairs'])}\n" - ) - - prompt = f""" -You are an AI Data Analyst. Summarize the dataset insights in simple, -professional language. Avoid technical jargon. Here is the analysis: - -Rows: {df.shape[0]} -Columns: {df.shape[1]} - -Numerical Columns: {stats['num_columns']} -Categorical Columns: {stats['cat_columns']} - -Missing Values: {stats['missing_values']} - -Strong Correlations (> 0.6): -{corr['high_correlation_pairs']} - -Give clear, meaningful insights suitable for a student competition project. -""" - - try: - model = genai.GenerativeModel("gemini-1.5-flash") - response = model.generate_content(prompt) - return response.text - except Exception: - return "Gemini failed to respond — using fallback rule-based insights." - -if __name__ == "__main__": - print("Running Analysis Agent...") - - # Path of your CSV - csv_path = "data\\data_science_student_marks.csv" - - # Load dataset - df = pd.read_csv(csv_path) - - # Compute stats - stats = compute_basic_stats(df) - - # Compute correlations - corr = compute_correlations(df) - - # Generate insights - insights = generate_ai_insights(df, stats, corr) - - # Final structured result: - result = { - "timestamp": str(datetime.now()), - "stats": stats, - "correlations": corr, - "ai_insights": insights, - } - - print("\n=== ANALYSIS RESULT ===") - print(result)