Metrics_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_df = pd.read_csv('data/deBERTA_result.csv')\n",
    "llama_df = pd.read_csv('data/llama_result.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_labels(lable):\n",
    "    lable = lable.strip().lower()\n",
    "    if 'unsafe' in lable: \n",
    "        return 1\n",
    "    elif 'safe' in lable and 'unsafe' not in lable: \n",
    "        return 0\n",
    "    if lable == \"injection\": return 1\n",
    "    return -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_df['Label'] = bert_df['Label'].apply(parse_labels)\n",
    "llama_df['Label'] = llama_df['Label'].apply(parse_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['Prompt', 'source', 'target', 'Label', 'Score'], dtype='object')\n",
      "Index(['Prompt', 'source', 'target', 'Label'], dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print(bert_df.columns)\n",
    "print(llama_df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to calculate metrics for each source\n",
    "def calculate_metrics_per_source(df, model_name):\n",
    "    results = []\n",
    "    grouped = df.groupby('source')\n",
    "    for source, group in grouped:\n",
    "        true_labels = group['target']\n",
    "        predicted_labels = group['Label']\n",
    "        \n",
    "        accuracy = accuracy_score(true_labels, predicted_labels)\n",
    "        precision = precision_score(true_labels, predicted_labels, zero_division=0)\n",
    "        recall = recall_score(true_labels, predicted_labels, zero_division=0)\n",
    "        f1 = f1_score(true_labels, predicted_labels, zero_division=0)\n",
    "        \n",
    "        results.append([source, accuracy, precision, recall, f1])\n",
    "    \n",
    "    # Create a DataFrame for the results\n",
    "    results_df = pd.DataFrame(results, columns=['Source', 'Accuracy', 'Precision', 'Recall', 'F1-score'])\n",
    "    \n",
    "    # Print the results in a compressed table format\n",
    "    print(f\"\\n{model_name} Model Metrics:\")\n",
    "    print(results_df.to_string(index=False, float_format=\"%.2f\"))\n",
    "    \n",
    "    # Calculate and print the average metrics\n",
    "    avg_metrics = results_df[['Accuracy', 'Precision', 'Recall', 'F1-score']].mean()\n",
    "    print(f\"\\nAverage Metrics for {model_name}:\")\n",
    "    print(avg_metrics.to_string(float_format=\"%.2f\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "BERT Model Metrics:\n",
      "              Source  Accuracy  Precision  Recall  F1-score\n",
      " forbidden_questions      1.00       1.00    1.00      1.00\n",
      "infected_given_train      0.49       1.00    0.49      0.66\n",
      "   jailbreak_prompts      0.77       1.00    0.77      0.87\n",
      "            leetcode      0.99       0.00    0.00      0.00\n",
      "     row_given_train      0.61       0.00    0.00      0.00\n",
      "\n",
      "Average Metrics for BERT:\n",
      "Accuracy    0.77\n",
      "Precision   0.60\n",
      "Recall      0.45\n",
      "F1-score    0.51\n"
     ]
    }
   ],
   "source": [
    "# Calculate metrics for each source in BERT\n",
    "calculate_metrics_per_source(bert_df, \"BERT\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "LLaMA Model Metrics:\n",
      "              Source  Accuracy  Precision  Recall  F1-score\n",
      "infected_given_train      0.10       1.00    0.10      0.19\n",
      "   jailbreak_prompts      0.30       1.00    0.30      0.46\n",
      "     row_given_train      0.99       0.00    0.00      0.00\n",
      "\n",
      "Average Metrics for LLaMA:\n",
      "Accuracy    0.47\n",
      "Precision   0.67\n",
      "Recall      0.14\n",
      "F1-score    0.22\n"
     ]
    }
   ],
   "source": [
    "# Calculate metrics for each source in LLaMA\n",
    "calculate_metrics_per_source(llama_df, \"LLaMA\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}