diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 1c2b536..be1f8af 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -6,6 +6,10 @@ on: - main - master +permissions: + contents: read + pull-requests: write + jobs: label: runs-on: ubuntu-latest diff --git a/agents/analysis_agent.py b/agents/analysis_agent.py deleted file mode 100644 index 6efe3a2..0000000 --- a/agents/analysis_agent.py +++ /dev/null @@ -1,145 +0,0 @@ -# agents/analysis_agent.py - -#Analysis Agent for ProdigyFlow | Performs simple EDA and generates AI insights using Google ADK. - - -import pandas as pd -import os -import json -from pathlib import Path -import re - -try: - import google.generativeai as genai - genai.configure(api_key=os.getenv("GOOGLE_API_KEY")) - ADK_AVAILABLE = True -except Exception: - ADK_AVAILABLE = False - -def generate_adk_insights(text: str) -> str: - if not ADK_AVAILABLE: - return "AI summary unavailable (ADK not installed)." - - try: - model = genai.GenerativeModel("gemini-2.0-flash") - prompt = ( - "Convert the dataset insights into clean, simple bullet points. " - "Do not use markdown symbols like ** or *. Be concise.\n\n" - f"Insights:\n{text}" - ) - - response = model.generate_content(prompt) - - raw = response.text.replace("\\n", "\n").strip() - - cleaned = raw.replace("**", "").replace("* ", "").replace("* ", "") - - lines = cleaned.splitlines() - normalized = [] - for line in lines: - l = line.strip() - if re.match(r"^[-ā€¢]\s+", l): - l = "- " + l.split(maxsplit=1)[1] - else: - if l: - l = "- " + l - l = re.sub(r"^(-\s*)+", "- ", l) - - normalized.append(l) - if normalized and "key findings" in normalized[0].lower(): - normalized = normalized[1:] - - return "\n".join(line for line in normalized if line.strip()) - - except Exception as e: - return f"AI summary unavailable ({str(e)})." - - -# ------------------------- -# Main Analysis Function -# ------------------------- -def analyze(cleaned_csv_path: str, logger=None): - if logger: - logger.info("Loading cleaned dataset...") - - df = pd.read_csv(cleaned_csv_path) - df.fillna("ā€”", inplace=True) - - num_rows, num_cols = df.shape - - missing_count = df.isna().sum().to_dict() - missing_percent = (df.isna().mean() * 100).round(2).to_dict() - - describe_stats = df.describe(include="all").to_dict() - - # Correlations - try: - corr_matrix = df.corr(numeric_only=True).round(3).to_dict() - except: - corr_matrix = {} - - insights = { - "dataset_overview": { - "rows": num_rows, - "columns": num_cols, - "column_names": list(df.columns), - }, - "missing_values": { - "count": missing_count, - "percent": missing_percent, - }, - "summary_statistics": describe_stats, - "correlation_matrix": corr_matrix, - } - - # AI summary - adk_input_text = json.dumps(insights, indent=2) - insights["ai_summary"] = generate_adk_insights(adk_input_text) - - metadata = { - "status": "analysis_complete", - "num_numeric_columns": len(df.select_dtypes(include="number").columns), - "used_adk": ADK_AVAILABLE, - } - - if logger: - logger.info("Analysis completed successfully.") - - return insights, metadata - -BOLD = "\033[1m" -RESET = "\033[0m" -CYAN = "\033[96m" - -def hr(title: str): - print(f"\n{BOLD}{title}{RESET}") - print("-" * len(title)) - - -if __name__ == "__main__": - print(f"šŸš€ {BOLD}Running a dry test of analysis_agent...{RESET}\n") - - sample_path = "data\\data_science_student_marks.csv" - sample_path = str(Path(sample_path).resolve()) - - print(f"šŸ“‚ Using file: {sample_path}") - - try: - insights, meta = analyze(sample_path) - - insights_no_ai = dict(insights) - insights_no_ai.pop("ai_summary", None) - - hr("šŸ“Š INSIGHTS (Structured Data Overview)") - print(json.dumps(insights_no_ai, indent=2)) - - hr("šŸ¤– AI-GENERATED SUMMARY") - print(insights["ai_summary"]) - - hr("šŸ“ METADATA") - print(json.dumps(meta, indent=2)) - - print("\nāœ” Analysis completed successfully!\n") - - except FileNotFoundError: - print("Sample file not found. Dry run complete.\n") diff --git a/agents/cleaning_agent.py b/agents/cleaning_agent.py index e69de29..5e56018 100644 --- a/agents/cleaning_agent.py +++ b/agents/cleaning_agent.py @@ -0,0 +1,153 @@ +import pandas as pd +import os +import google.generativeai as genai + +# Initialize Gemini client +# Note: Ensure GEMINI_API_KEY is set in your environment variables +try: + api_key = os.environ.get("GEMINI_API_KEY") + if api_key: + genai.configure(api_key=api_key) + model = genai.GenerativeModel('gemini-1.5-flash') + print("Gemini client initialized successfully.") + else: + print("Warning: GEMINI_API_KEY not found in environment variables. LLM features will be disabled.") + model = None +except Exception as e: + print(f"Warning: Gemini client could not be initialized. Error: {e}") + model = None + +def load_data(filepath): + """Loads data from a CSV file.""" + try: + df = pd.read_csv(filepath) + print(f"Successfully loaded data from {filepath}") + return df + except FileNotFoundError: + print(f"Error: File not found at {filepath}") + return None + except Exception as e: + print(f"Error loading data: {e}") + return None + +def save_data(df, filepath): + """Saves the DataFrame to a CSV file.""" + try: + df.to_csv(filepath, index=False) + print(f"Successfully saved cleaned data to {filepath}") + except Exception as e: + print(f"Error saving data: {e}") + +def clean_missing(df, strategy='ffill'): + """ + Handle missing values. + Strategies: 'ffill', 'bfill', 'drop', 'mean', 'median', 'mode' + """ + print(f"Executing clean_missing with strategy: {strategy}") + if strategy == 'ffill': + return df.ffill() + elif strategy == 'bfill': + return df.bfill() + elif strategy == 'drop': + return df.dropna() + elif strategy == 'mean': + return df.fillna(df.mean(numeric_only=True)) + elif strategy == 'median': + return df.fillna(df.median(numeric_only=True)) + elif strategy == 'mode': + return df.fillna(df.mode().iloc[0]) + else: + print(f"Unknown strategy '{strategy}', defaulting to ffill") + return df.ffill() + +def clean_duplicates(df, keep='first'): + """Removes duplicate rows.""" + print(f"Executing clean_duplicates keeping: {keep}") + return df.drop_duplicates(keep=keep) + +def clean_text(df): + """Standardizes text columns (strip whitespace, lowercase).""" + print("Executing clean_text") + for col in df.select_dtypes(include='object'): + df[col] = df[col].str.strip().str.lower() + return df + +def fix_dtypes(df): + """Infers better data types for columns.""" + print("Executing fix_dtypes") + return df.infer_objects() + +def get_cleaning_plan(df_head_str, user_prompt): + """Uses Gemini to suggest a cleaning plan based on data sample and user request.""" + if not model: + return "missing, duplicate, text, datatype" # Fallback default plan + + system_prompt = """You are a data-cleaning agent. + Analyze the provided data sample and user request. + Return a comma-separated list of cleaning steps to apply. + Available steps: 'missing', 'duplicate', 'text', 'datatype'. + Example output: missing, text + """ + + content = f"{system_prompt}\n\nData Sample:\n{df_head_str}\n\nUser Request: {user_prompt}" + + try: + response = model.generate_content(content) + return response.text.strip() + except Exception as e: + print(f"Error getting plan from LLM: {e}") + return "missing, duplicate, text, datatype" + +def clean_dataset(df, plan): + """Applies cleaning functions based on the plan.""" + plan = plan.lower() + + if "missing" in plan: + # Simple heuristic: if numeric, use mean, else ffill + # For now, defaulting to ffill/bfill as per original or user preference could be added + df = clean_missing(df) + if "duplicate" in plan: + df = clean_duplicates(df) + if "text" in plan: + df = clean_text(df) + if "datatype" in plan: + df = fix_dtypes(df) + + return df + +def analyze_data(df): + """Prints basic analysis of the dataset.""" + print("\n--- Data Analysis ---") + print(f"Shape: {df.shape}") + print(f"Missing Values:\n{df.isnull().sum()}") + print(f"Duplicates: {df.duplicated().sum()}") + print("---------------------\n") + +def main(): + input_file = "data//data_science_student_marks.csv" + output_file = "cleaned_data.csv" + + print("--- Data Cleaning Agent (Gemini Powered) ---") + df = load_data(input_file) + + if df is not None: + analyze_data(df) + + user_prompt = "Clean this dataset automatically." # Default prompt + # In a real interactive CLI, we would ask: input("Enter your cleaning goal: ") + + print("Generating cleaning plan...") + plan = get_cleaning_plan(df.head().to_string(), user_prompt) + print(f"Proposed Plan: {plan}") + + # In interactive mode, we would ask for confirmation here. + print("Applying plan...") + cleaned_df = clean_dataset(df, plan) + + analyze_data(cleaned_df) + + save_data(cleaned_df, output_file) + print("Done.") + +if __name__ == "__main__": + main()