Chapter_12.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Multi-class Conformal Prediction"
      ],
      "metadata": {
        "id": "Mj5QB2clnWeL"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "In this notebook we will look at various methods for calibrating class probabilities in multi-class classification problems.\n"
      ],
      "metadata": {
        "id": "XD-ysS74p99Y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install --upgrade venn-abers\n",
        "\n",
        "!pip install catboost\n",
        "!pip install dtype_diet\n",
        "\n",
        "#install OpenML\n",
        "!pip install openml"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0LcO8DJ3rptf",
        "outputId": "dbb41dfb-1571-4c6b-f401-c74d6ba8518c"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: venn-abers in /usr/local/lib/python3.10/dist-packages (1.3)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from venn-abers) (1.23.5)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from venn-abers) (1.2.2)\n",
            "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->venn-abers) (1.11.2)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->venn-abers) (1.3.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->venn-abers) (3.2.0)\n",
            "Requirement already satisfied: catboost in /usr/local/lib/python3.10/dist-packages (1.2.2)\n",
            "Requirement already satisfied: graphviz in /usr/local/lib/python3.10/dist-packages (from catboost) (0.20.1)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from catboost) (3.7.1)\n",
            "Requirement already satisfied: numpy>=1.16.0 in /usr/local/lib/python3.10/dist-packages (from catboost) (1.23.5)\n",
            "Requirement already satisfied: pandas>=0.24 in /usr/local/lib/python3.10/dist-packages (from catboost) (1.5.3)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from catboost) (1.11.2)\n",
            "Requirement already satisfied: plotly in /usr/local/lib/python3.10/dist-packages (from catboost) (5.15.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from catboost) (1.16.0)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=0.24->catboost) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=0.24->catboost) (2023.3.post1)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->catboost) (1.1.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->catboost) (0.11.0)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->catboost) (4.42.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->catboost) (1.4.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->catboost) (23.1)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->catboost) (9.4.0)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->catboost) (3.1.1)\n",
            "Requirement already satisfied: tenacity>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from plotly->catboost) (8.2.3)\n",
            "Requirement already satisfied: dtype_diet in /usr/local/lib/python3.10/dist-packages (0.0.2)\n",
            "Requirement already satisfied: pandas>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from dtype_diet) (1.5.3)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.10/dist-packages (from dtype_diet) (0.9.0)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0.0->dtype_diet) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0.0->dtype_diet) (2023.3.post1)\n",
            "Requirement already satisfied: numpy>=1.21.0 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0.0->dtype_diet) (1.23.5)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas>=1.0.0->dtype_diet) (1.16.0)\n",
            "Requirement already satisfied: openml in /usr/local/lib/python3.10/dist-packages (0.14.1)\n",
            "Requirement already satisfied: liac-arff>=2.4.0 in /usr/local/lib/python3.10/dist-packages (from openml) (2.5.0)\n",
            "Requirement already satisfied: xmltodict in /usr/local/lib/python3.10/dist-packages (from openml) (0.13.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from openml) (2.31.0)\n",
            "Requirement already satisfied: scikit-learn>=0.18 in /usr/local/lib/python3.10/dist-packages (from openml) (1.2.2)\n",
            "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.10/dist-packages (from openml) (2.8.2)\n",
            "Requirement already satisfied: pandas>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from openml) (1.5.3)\n",
            "Requirement already satisfied: scipy>=0.13.3 in /usr/local/lib/python3.10/dist-packages (from openml) (1.11.2)\n",
            "Requirement already satisfied: numpy>=1.6.2 in /usr/local/lib/python3.10/dist-packages (from openml) (1.23.5)\n",
            "Requirement already satisfied: minio in /usr/local/lib/python3.10/dist-packages (from openml) (7.1.17)\n",
            "Requirement already satisfied: pyarrow in /usr/local/lib/python3.10/dist-packages (from openml) (9.0.0)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0.0->openml) (2023.3.post1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil->openml) (1.16.0)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.18->openml) (1.3.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.18->openml) (3.2.0)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from minio->openml) (2023.7.22)\n",
            "Requirement already satisfied: urllib3 in /usr/local/lib/python3.10/dist-packages (from minio->openml) (2.0.4)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->openml) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->openml) (3.4)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import torch\n",
        "\n",
        "import openml\n",
        "\n",
        "import time\n",
        "\n",
        "import plotly.io as pio\n",
        "pio.renderers.default = 'colab'\n",
        "\n",
        "# Set the style for visualization\n",
        "sns.set_style(\"whitegrid\")\n",
        "\n",
        "from tqdm import tqdm\n",
        "\n",
        "from dtype_diet import report_on_dataframe, optimize_dtypes\n",
        "\n",
        "import plotly.graph_objs as go\n",
        "import plotly.figure_factory as ff\n",
        "from plotly import tools\n",
        "from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\n",
        "from plotly.subplots import make_subplots\n",
        "init_notebook_mode(connected=True)\n",
        "\n",
        "import gc\n",
        "from datetime import datetime\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.model_selection import KFold\n",
        "\n",
        "from sklearn.preprocessing import LabelBinarizer\n",
        "\n",
        "from sklearn.metrics import accuracy_score, balanced_accuracy_score, precision_score, average_precision_score, recall_score, f1_score, roc_auc_score, log_loss, brier_score_loss\n",
        "from sklearn.calibration import CalibrationDisplay, calibration_curve\n",
        "from matplotlib.gridspec import GridSpec\n",
        "\n",
        "from sklearn.isotonic import IsotonicRegression\n",
        "from sklearn.calibration import CalibratedClassifierCV\n",
        "\n",
        "\n",
        "from sklearn.dummy import DummyClassifier\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.ensemble import AdaBoostClassifier\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "\n",
        "from sklearn.model_selection import cross_val_predict\n",
        "\n",
        "from catboost import CatBoostClassifier\n",
        "from sklearn import svm\n",
        "from lightgbm import LGBMClassifier\n",
        "from sklearn.svm import SVC, LinearSVC\n",
        "from xgboost import XGBClassifier\n",
        "\n",
        "from venn_abers import VennAbersCalibrator\n",
        "\n",
        "from scipy.io import arff\n",
        "\n",
        "pd.set_option('display.max_columns', 100)\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "VFuZxn3BnbfP",
        "outputId": "807b81c7-a602-4986-9e27-c96c774dc7db"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "        <script type=\"text/javascript\">\n",
              "        window.PlotlyConfig = {MathJaxConfig: 'local'};\n",
              "        if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
              "        if (typeof require !== 'undefined') {\n",
              "        require.undef(\"plotly\");\n",
              "        requirejs.config({\n",
              "            paths: {\n",
              "                'plotly': ['https://cdn.plot.ly/plotly-2.24.1.min']\n",
              "            }\n",
              "        });\n",
              "        require(['plotly'], function(Plotly) {\n",
              "            window._Plotly = Plotly;\n",
              "        });\n",
              "        }\n",
              "        </script>\n",
              "        "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check if CUDA (GPU support) is available\n",
        "is_cuda_available = torch.cuda.is_available()\n",
        "\n",
        "print(f\"CUDA (GPU support) is available: {is_cuda_available}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F4KsoZBKUGlA",
        "outputId": "2943fcc8-957e-4c8c-d632-34891b910a94"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CUDA (GPU support) is available: False\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Load dataset"
      ],
      "metadata": {
        "id": "AEZYZo4kskf1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# List datasets https://docs.openml.org/Python-API/\n",
        "datasets_df = openml.datasets.list_datasets(output_format=\"dataframe\")\n",
        "print(datasets_df.head(n=10))\n",
        "\n",
        "datasets_df.set_index('did', inplace = True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hScGN33irr-o",
        "outputId": "62a6c3c4-e7f8-405b-81b1-f368059109c3"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "    did             name  version uploader  status format  MajorityClassSize  \\\n",
            "2     2           anneal        1        1  active   ARFF              684.0   \n",
            "3     3         kr-vs-kp        1        1  active   ARFF             1669.0   \n",
            "4     4            labor        1        1  active   ARFF               37.0   \n",
            "5     5       arrhythmia        1        1  active   ARFF              245.0   \n",
            "6     6           letter        1        1  active   ARFF              813.0   \n",
            "7     7        audiology        1        1  active   ARFF               57.0   \n",
            "8     8  liver-disorders        1        1  active   ARFF                NaN   \n",
            "9     9            autos        1        1  active   ARFF               67.0   \n",
            "10   10            lymph        1        1  active   ARFF               81.0   \n",
            "11   11    balance-scale        1        1  active   ARFF              288.0   \n",
            "\n",
            "    MaxNominalAttDistinctValues  MinorityClassSize  NumberOfClasses  \\\n",
            "2                           7.0                8.0              5.0   \n",
            "3                           3.0             1527.0              2.0   \n",
            "4                           3.0               20.0              2.0   \n",
            "5                          13.0                2.0             13.0   \n",
            "6                          26.0              734.0             26.0   \n",
            "7                          24.0                1.0             24.0   \n",
            "8                           NaN                NaN              0.0   \n",
            "9                          22.0                3.0              6.0   \n",
            "10                          8.0                2.0              4.0   \n",
            "11                          3.0               49.0              3.0   \n",
            "\n",
            "    NumberOfFeatures  NumberOfInstances  NumberOfInstancesWithMissingValues  \\\n",
            "2               39.0              898.0                               898.0   \n",
            "3               37.0             3196.0                                 0.0   \n",
            "4               17.0               57.0                                56.0   \n",
            "5              280.0              452.0                               384.0   \n",
            "6               17.0            20000.0                                 0.0   \n",
            "7               70.0              226.0                               222.0   \n",
            "8                6.0              345.0                                 0.0   \n",
            "9               26.0              205.0                                46.0   \n",
            "10              19.0              148.0                                 0.0   \n",
            "11               5.0              625.0                                 0.0   \n",
            "\n",
            "    NumberOfMissingValues  NumberOfNumericFeatures  NumberOfSymbolicFeatures  \n",
            "2                 22175.0                      6.0                      33.0  \n",
            "3                     0.0                      0.0                      37.0  \n",
            "4                   326.0                      8.0                       9.0  \n",
            "5                   408.0                    206.0                      74.0  \n",
            "6                     0.0                     16.0                       1.0  \n",
            "7                   317.0                      0.0                      70.0  \n",
            "8                     0.0                      6.0                       0.0  \n",
            "9                    59.0                     15.0                      11.0  \n",
            "10                    0.0                      3.0                      16.0  \n",
            "11                    0.0                      4.0                       1.0  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#plot number of datasets for each class size where class size is more than 2\n",
        "datasets_df[datasets_df['NumberOfClasses'] > 2]['NumberOfClasses'].value_counts()[:20].plot(kind = 'bar')\n",
        "plt.title('Count of datasets with various number of classes in OpenML')\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 469
        },
        "id": "uxm-NyMBssiX",
        "outputId": "14b581b5-03a1-4b8c-f693-965c019491dc"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAHECAYAAAAaiHGbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABHuElEQVR4nO3dd3gUhfr+/ztZiqGFluABBA9wshiSQAIIBKQrkoACKggiVRCkWZAmggjSRI8UBQSkqRSlfORQRMSDiKCglICYQ+8CCUIICW0zvz/8Zb8uSSC72WQn2ffrurgudurz7M7O3tmZ2fExDMMQAACAifl6ugAAAIB7IbAAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7DkMrdv39bkyZPVqFEjVa1aVS+99JLTy2jatKmGDRuWDdV5p5UrV8pqter06dOZnjYmJiYHKsuaYcOGqWnTpp4uI9f46aefZLVatWHDBk+XkilxcXEaOHCg6tSpI6vVqgULFmRpeXl9v+LM+xzZI1cGlpMnT2rUqFFq1qyZQkNDFRERoWeffVYLFy7U9evXPV2eJOmzzz7TypUr3b7cFStWaN68eWrRooUmTpyobt26uX0dGTl//rymT5+ugwcP5tg672bLli2aPn26p8tIV3a9/oC7TJgwQVu3blXv3r01efJkPfLII54uySslJSXpww8/VOvWrVW9enXVrFlTnTp10urVq2W2O+c8//zzslqteuyxx9Idv23bNlmt1jTB3V1/qOXL0twe8N///leDBg1SgQIF9OSTTyooKEi3bt3SL7/8onfffVeHDx/W2LFjPV2mlixZohIlSqhdu3ZuXe6OHTtUpkwZjRgxwq3LzYwLFy5oxowZKleunB566KEcX/+dtmzZos8++0wDBgzwaB1PPvmkoqOjVaBAAfuw7Hr9c9LYsWNNt8OE++zYsUPNmjVTz549PV1KrpDe+zyr4uLi1K1bNx05ckRRUVHq3Lmzbty4oY0bN2ro0KHasmWLpkyZIovF4rZ1ZlXBggV14sQJ7du3T2FhYQ7j1qxZo4IFC+rGjRvZsu5cFVhOnTqlV155RWXLltXChQsVGBhoH/fcc8/pxIkT+u9//+u5AnNAfHy8ihUr5uky8DcWi8VUO5SsSkpKUqFChZQ/f35Pl4J0pL4+WcW+xDnZ8T4fOnSojhw5ohkzZqhZs2b24V26dNGkSZP0ySef6KGHHlLv3r3dut6sqFChgm7fvq3//Oc/DoHlxo0b+uabb9S4cWN9/fXX2bLuXHVIaO7cuUpKStI777zjEFZSVaxYUV27drU/vn37tj788EM1b95cISEhatq0qd5//33dvHnTYT6r1ZruoYU7j8mmfq31yy+/aMKECapbt65q1Kihfv366dKlSw7zHTp0SD///LP967Hnn3/+rr0lJSVp4sSJatSokUJCQtSiRQvNmzfP/hfu6dOnZbVa9dNPP+nQoUP25f70008ZLtMwDH300Udq2LChqlevrueff16HDh1KM93ly5c1adIktW7dWuHh4YqIiNALL7yg33//3T7NTz/9pKefflqSNHz4cPv6Uw977Nq1SwMHDlTjxo0VEhKiRo0aafz48WkO0V28eFHDhw9Xw4YNFRISogYNGqhv375pjgtv2bJFnTp1Uo0aNRQeHq7evXs71D5s2DB99tlnkmSvxWq12sevXbtW7dq1s/fTunVrLVy48K6vQdu2bdW/f3+HYa1bt5bVanV4LtatWyer1aojR45ISntsOzOv/82bN++6DaVn3rx5slqtOnPmTJpx7733nkJCQnTlyhVJmX89hg0bpvDwcJ08eVK9evVSeHi4Bg8ebB935zks99pOpf+3raZ3SOzO91piYqLeeecdNW3aVCEhIapXr566d++uAwcO3PW5mD59uqxWq06cOKFhw4apVq1aqlmzpoYPH67k5GSXakld5rFjxzR48GDVrFlTdevW1QcffCDDMHTu3Dn17dtXERERql+/vj755JN0a0tJSdH777+v+vXrq0aNGurTp4/OnTuXZrq9e/eqZ8+eqlmzpqpXr67OnTvrl19+SbfPw4cP67XXXlPt2rXVqVOnuz43p06d0sCBA/Xwww+revXqat++vcMfcqnbq2EY+uyzz9K8dzLqaeHChWrdurVCQ0NVt25d9ezZ865f8Wdmv5Jq8eLFio6OVvXq1VW7dm21a9dOa9assY/P7HaSmefU1W0uvXNYmjZtqhdffFG7du3S008/rdDQUDVr1kyrV6++67Ikac+ePfrhhx/Utm1bh7CS6rXXXtODDz6ouXPn2t+3qdvzvHnztGDBAjVp0kRhYWHq3Lmz/ve//6VZxpEjR+zbQmhoqNq1a6dvv/023b7u9bn2d61atdK6deuUkpJiH7Z582Zdv35djz/++D17d1Wu+oblu+++0wMPPKCIiIhMTT9y5EitWrVKLVq0UPfu3bVv3z7Nnj1bR44c0YcffuhyHePGjVOxYsXUv39/nTlzRgsXLtTbb7+tDz74QJI0YsQIjR07VoUKFVKfPn0kSaVLl85weYZhqG/fvvZQ8NBDD2nr1q2aPHmyzp8/rxEjRqhkyZKaPHmyZs2apaSkJL366quSpMqVK2e43KlTp2rmzJlq1KiRGjVqpAMHDqhHjx66deuWw3SnTp3Spk2b9Pjjj6t8+fKKi4vTsmXL1LlzZ61du1ZlypRR5cqVNXDgQE2bNk0dOnRQzZo1Jcn+WmzYsEHXr19Xx44dVbx4ce3bt0+ffvqp/vjjD02bNs2+rgEDBujw4cPq3LmzypUrp0uXLmnbtm06d+6cypcvL0lavXq1hg0bpgYNGmjw4MFKTk7WkiVL1KlTJ61atUrly5dXhw4ddOHCBW3btk2TJ0926Gfbtm169dVXVa9ePfuH79GjR/Xrr786BNo71axZU2vXrrU/vnz5sg4dOiRfX1/98ssvqlq1qqS/wkDJkiUzfO4z8/rfaxtKT8uWLfXuu+9q/fr1euGFFxzGrV+/XvXr15e/v79Tr4f0V7BP3ckPHTpU9913X7rrz8x26qzRo0fr66+/VufOnVW5cmVdvnxZv/zyi44cOaJq1ardc/6XX35Z5cuX16uvvqrffvtNX3zxhUqWLKnXX3/d6VpSvfLKK6pcubJee+01bdmyRTNnzlTx4sW1dOlS1a1bV4MHD9aaNWs0adIkhYaGqnbt2g7zz5w5Uz4+PurVq5fi4+O1cOFCdevWTf/3f/9nf263b9+uXr16KSQkRP3795ePj49Wrlyprl276vPPP0/zVfugQYNUsWJFvfLKK3c9TBcXF6dnn31WycnJev7551WiRAmtWrVKffv21bRp0/Too4+qdu3amjx5soYMGaL69evrySefvOdz8sYbb2jlypVq2LChnn76adlsNu3atUt79+5VaGhouvNkZr8iScuXL9e4cePUokULdenSRTdu3FBsbKz27t2r1q1bS8rcdpLZ5zSr29ydTpw4oUGDBunpp59W27ZttWLFCg0bNkzVqlXTv/71rwzn++677yRJbdq0SXd8vnz51KpVK82YMUO//vqrIiMj7eNWr16ta9euqVOnTrpx44YWL16srl27as2aNfZ9zaFDh9SxY0eVKVNGvXr1UqFChbR+/Xr169dP06dP16OPPuqwPmf2Sa1atdL06dP1008/qV69epKk//znP6pbt65KlSrlzNPnHCOXuHr1qhEUFGT07ds3U9MfPHjQCAoKMt544w2H4RMnTjSCgoKM7du324cFBQUZ06ZNS7OMJk2aGEOHDrU/XrFihREUFGR069bNSElJsQ8fP3688dBDDxkJCQn2YdHR0Ubnzp0zVes333xjBAUFGR999JHD8AEDBhhWq9U4ceKEfVjnzp2N6Ojoey4zPj7eqFatmtG7d2+HWt9//30jKCjIoa8bN24YNpvNYf5Tp04ZISEhxowZM+zD9u3bZwQFBRkrVqxIs77k5OQ0w2bPnm1YrVbjzJkzhmEYxpUrV4ygoCBj7ty5GdadmJho1KpVyxg5cqTD8IsXLxo1a9Z0GD5mzBgjKCgozTLGjRtnREREGLdv385wPelZv369ERQUZBw+fNgwDMP49ttvjZCQEKNPnz7Gyy+/bJ+udevWRr9+/eyPU7eLU6dO2Ydl9Po7sw2lp0OHDkbbtm0dhu3du9cICgoyVq1aZR+WmdfDMAxj6NChRlBQkDFlypQ00w8dOtRo0qSJ/XFmt9NTp05luJ3c+V6rWbOmMWbMmLv2nJ5p06YZQUFBxvDhwx2G9+vXz3j44Yftj52pJXWZb775pn3Y7du3jYYNGxpWq9WYPXu2ffiVK1eMsLAwh/fRjh07jKCgIOORRx4xrl69ah++bt06IygoyFi4cKFhGIaRkpJiPPbYY0aPHj0ctoHk5GSjadOmRvfu3dPU9Oqrr2bqeXnnnXeMoKAgY+fOnfZhiYmJRtOmTY0mTZo4vM+DgoIy9dxv377dCAoKMsaOHZtm3N/rv3N/mdn9St++fe+5T7vXduLMc+rqNpfe+7xJkyZpnu/4+HgjJCTEmDhx4l2X99JLLxlBQUHGlStXMpxm48aNRlBQkLFo0SLDMP7f9hwWFmb88ccf9ulS9wHjx4+3D+vatavRqlUr48aNG/ZhKSkpRocOHYzHHnssTV+Z2Sf9/fOnXbt2xogRIwzD+Ov9UK1aNWPVqlX298H69evTrGPfvn13fU7uJdccEkpMTJQkFS5cOFPTb9myRZLUvXt3h+E9evRwGO+K9u3by8fHx/64Vq1astls6X5Vnxnff/+9LBZLmsMGPXr0kGEY+v77751e5o8//qhbt26pc+fODrWm9w1DgQIF5Ov716Zgs9n0559/qlChQvrnP/+p3377LVPr+/tf5UlJSbp06ZLCw8NlGIZ9Gffdd5/y58+vn3/+2X7oIr26ExISFB0drUuXLtn/+fr6qnr16nc9BJaqWLFiSk5O1rZt2zJVe6patWpJknbu3Cnpr29SQkNDVb9+fe3atUuSlJCQoEOHDtmndZWr21DLli114MABnTx50j5s/fr1KlCggJo3b24flpnX4+86dux4z5qzYzstVqyY9u7dq/Pnzzs9ryQ9++yzDo9r1aqly5cv2/cXrkg99Cn9dd5CSEiIDMNwGF6sWDH985//1KlTp9LM36ZNGxUpUsT++PHHH1dAQIB9n3Pw4EEdP35crVu31p9//mnfxpOSklSvXj3t3LnT4av29PrMyJYtWxQWFuawfRYuXFgdOnTQmTNndPjw4cw9CX+zceNG+fj4pDlcKslhG75TZvcrxYoV0x9//KF9+/ZluKx7bSfOPKdZ3ebuVKVKFYfnu2TJkhluG3937do1SXf/TEsdd+f23Lx5c/s3VJIUFham6tWr27exy5cva8eOHWrZsqUSExPtz8eff/6pBg0a6Pjx42n6d3af1Lp1a33zzTe6efOmvv76a1ksFod9UHbINYeEUncAqS/yvZw5c0a+vr6qUKGCw/CAgAAVK1bM5XAhSWXLlnV4nHriWkJCgkvLO3PmjAIDAx12ctL/O9zjSq1nz56VJD344IMOw0uWLGk/bJAqJSVFixYt0ueff67Tp0/LZrPZxxUvXjzT65s2bZo2b96cJoykvtkKFCigwYMHa9KkSapfv76qV6+uxo0bq02bNgoICJAkHT9+XFL6wUpSmucoPZ06ddL69evVq1cvlSlTRvXr11fLli3VsGHDu85XunRpPfjgg9q1a5eeffZZ/fLLL6pTp45q1aqlsWPH6tSpUzpy5IhSUlLsh8Rc5eo29Pjjj2vixIlat26d+vTpI8MwtGHDBjVs2NDhucnM65EqX758uv/+++9Zc3Zsp4MHD9awYcPUuHFjVatWTY0aNVKbNm30wAMPZGr+jJ7HK1euZGpbycwyixYtqoIFC6pkyZJphl++fDnN/BUrVnR47OPjo4oVK9qfn9RtfOjQoRnWcPXqVYf3aerh0ns5e/asqlevnmZ4pUqV7OODgoIytaxUJ0+eVGBgYKb3Bakyu1/p1auXfvzxRz3zzDOqWLGi6tevr1atWjm8x+61nTjznGZ1m7vTP/7xjzTD/P39M/yjLFVqGLl27VqGJz9nFGru3Makv/b169evl/TXa2YYhqZOnaqpU6emu+z4+HiH0OPsPikqKkqTJk3S999/r6+++kqNGzd2+T2XWbkqsAQGBqZ70ujd3O0vgHv5+xvs71L/ariTkUsvAZ01a5amTp2qp556SoMGDZK/v798fX01fvz4TPVks9nUvXt3XblyRS+88IIqVaqkQoUK6fz58xo2bJjDX4vdunVT06ZNtWnTJv3www+aOnWqPv74Yy1cuFDBwcH29U2ePNkeYv4uM2fplypVSqtXr9YPP/yg77//Xt9//71WrlypNm3aaNKkSXedNyIiQjt27ND169d14MABvfTSSwoKClKxYsW0a9cuHTlyRIUKFVJwcPA967gbV7ehMmXKqFatWlq/fr369OmjPXv26OzZs/ZzdSTnXg/J8S9hd8joPZfe+ykqKkq1atXSN998o23btmnevHmaM2eOpk+frkaNGt1zXfd6Hp2p5W7LzGi7c+U9nzrPkCFDMvx5gDuvAipYsKDT6/G0zO5XKleurA0bNui///2vtm7dqo0bN+rzzz9Xv379NHDgQEn33k6ceU6zus3dydUrhypXrqxNmzYpNjY2zXlQqWJjYyX99S2OM1Lf4z169Mjw93Xu/GPe2X1SYGCgHn74Yc2fP1+//vprjvwmVq4JLJLUpEkTLVu2TLt371Z4ePhdpy1XrpxSUlJ04sQJh5Mj4+LilJCQoHLlytmH+fv7p0mRN2/e1MWLF12u1ZmgVK5cOW3fvl2JiYkOCfXo0aP28c5KTcvHjx93+Mvh0qVLaZL/119/rTp16mj8+PEOwxMSElSiRAn744x6+t///qfjx49r0qRJDieQZXRIpkKFCurRo4d69Oih48ePq02bNvrkk080ZcoUe62lSpVyOMksPff6Orpp06Zq2rSpUlJS9NZbb2nZsmV66aWX0v3rJFWtWrW0cuVKrV27VjabTREREfL19VXNmjXtgSUiIuKeO6msBOV7admypcaMGaOjR49q3bp18vPzU5MmTezjnX09Miuz22nqNwN3vqdSv/W7U2BgoJ577jk999xzio+PV9u2bTVr1iyXPjzu5Gwt7nDixAmHx4Zh6MSJE/YrcVK38SJFitxzG3dW2bJldezYsTTDU1+jO/+KzowKFSrohx9+0OXLl536liWz+xXprzARFRWlqKgo3bx5UwMGDNCsWbP04osv2sPa3bYTZ5/T7NzmMqtx48aaPXu2Vq9enW5gsdlsWrNmjfz9/dNcaHLnNib9ta9PfQ+mPh/58+d3+zb2d61atdLIkSNVrFixe36D7Q655hwWSXrhhRdUqFAhjRw5UnFxcWnGnzx50n7pauqGd+elrPPnz3cYL/314qaeo5Bq+fLld/0r7F78/PwyfYioYcOGstls9st0Uy1YsEA+Pj4ubQiRkZHKnz+/Pv30U4eEnN6lvRaLJU2KXr9+fZpjnH5+fpLS7vxTk/nfl2EYhhYtWuQwXXJycpofFKpQoYIKFy5sv9T8kUceUZEiRTR79uw0VzNJcrjMLqN6/vzzzzT1pX5Y3HlJ+51Sj0XPmTNHVqtVRYsWlfTXFUTbt2/X/v37M3U4yJnX31ktWrSQxWLR2rVrtWHDBjVu3NjhL/LMvh7Oyux2WqRIEZUoUSLNe+rzzz93eGyz2XT16lWHYaVKlVJgYOA9X6fMymwt7rR69WqHw24bNmzQxYsX7c9PSEiIKlSooE8++STdQ9z3urz9bho1aqR9+/Zp9+7d9mFJSUlavny5ypUr5/Rf6pL02GOPyTAMzZgxI824u33DlNn9yp3v1wIFCqhy5coyDEO3bt3K1HaS2ec0J7a5zIqIiFBkZKRWrlxpv2Lo7/7973/r+PHjeuGFF9Jcubdp0yaH53Hfvn3au3evfRsrVaqUHn74YS1btkwXLlxIs+ysbGN/9/jjj6t///4aPXq0W39QLyO56huWChUqaMqUKXrllVcUFRVl/6Xbmzdvavfu3dqwYYP9l0WrVq2qtm3batmyZUpISFDt2rUVExOjVatWqXnz5qpbt659uc8884xGjx6tAQMGKDIyUr///rt++OGHNH8FOKNatWpasmSJPvroI1WsWFElS5a0X/51p6ZNm6pOnTr697//rTNnzshqtWrbtm369ttv1bVr1zRf3WVGyZIl1aNHD82ePVsvvviiGjVqpN9++03ff/99mr4aN26sDz/8UMOHD1d4eLj+97//ac2aNWmO6VaoUEHFihXT0qVLVbhwYRUqVEhhYWGqVKmSKlSooEmTJun8+fMqUqSIvv766zQf2MePH1e3bt30+OOPq0qVKrJYLNq0aZPi4uIUHR0t6a8PmLfeektDhgxRu3btFBUVpZIlS+rs2bPasmWLIiIiNGrUKPtzLP11OV6DBg1ksVgUHR2tkSNH6sqVK6pbt67KlCmjs2fP6tNPP9VDDz1018vApb+ODQcEBOjYsWMOJ5fWrl1bU6ZMkaRMnXDrzOvvrFKlSqlOnTqaP3++rl27pqioKIfxmX09nOXMdvrMM8/o448/1htvvKGQkBDt2rUrzV/+165dU6NGjdSiRQtVrVpVhQoV0o8//qiYmBi33pMmM7W4k7+/vzp16qR27drZL2uuWLGi2rdvL+mvQDlu3Dj16tVLrVq1Urt27VSmTBmdP39eP/30k4oUKaJZs2a5tO7evXtr7dq16tWrl55//nn5+/tr9erVOn36tKZPn+7Sob+6devqySef1OLFi3XixAk98sgjSklJsZ/j1blz53Tny+x+pWfPnipdurQiIiJUqlQpHT16VJ9++qkaNWqkIkWKKCEh4Z7bSWaf05za5jJr0qRJ6tatm1566SW1atVKtWrV0s2bN7Vx40b9/PPPioqKSveXiCtUqKCOHTuqY8eOunnzphYtWqTixYs7/NzB6NGj1alTJ7Vu3Vrt27fXAw88oLi4OO3Zs0d//PGHvvrqqyzXX7RoUad+aXzFihXaunVrmuFdunTJ1PkvuSqwSFKzZs301Vdfad68efr222+1ZMkSFShQQFarVcOGDbPvFKS/PsjKly+vVatWadOmTSpdurRefPHFNGe7t2/fXqdPn9aXX36prVu3qmbNmpo/f36W7tPTr18/nT17VnPnztW1a9f08MMPZ/iB5evrq5kzZ2ratGlat26dVq5cqXLlymnIkCH2q5pc8fLLL6tAgQJaunSpfvrpJ4WFhemTTz7Riy++6DBdnz59lJycrDVr1mjdunUKDg7W7Nmz9d577zlMlz9/fk2cOFHvv/++3nrrLd2+fVsTJkxQu3btNGvWLI0bN06zZ89WwYIF9eijj+q5555z+I2H+++/X9HR0dq+fbu++uorWSwWVapUSR988IFatGhhn65169YKDAzUxx9/rHnz5unmzZv2czf+/lP3jz32mJ5//nmtXbtWX331lQzDUHR0tJ544gktX75cn3/+uRISEhQQEKCWLVtqwIABmdph16xZUxs2bHD4GrZatWry8/PT7du30z2p8U7OvP6uiIqK0o8//qjChQun+Ro7f/78mXo9nOXMdpr6o1Nff/211q9fr4YNG2ru3LkOz8F9992njh07atu2bdq4caMMw1CFChXsO1p3yUwt7tSnTx/Fxsbq448/1rVr11SvXj2NHj3a/o2gJNWpU0fLli3TRx99pE8//VRJSUkKCAhQWFiYOnTo4PK6S5curaVLl+rdd9/Vp59+qhs3bshqtWrWrFlq3Lixy8udMGGCrFarvvzyS02ePFlFixZVSEjIXQ/NZ3a/0qFDB61Zs0bz589XUlKS7r//fj3//PP2G7tmdjvJzHOaU9tcZgUGBuqLL77Q/PnztWHDBm3cuFEWi0VWq1UTJ05UmzZt0j283KZNG/n6+mrhwoWKj49XWFiY3nzzTYcfVK1SpYpWrFihGTNmaNWqVbp8+bJKliyp4OBg9evXLyfbtFuyZEm6w9u1a5epwOJj5NYzRQEA8CKnT59Ws2bNNGTIEK+8B1SuOocFAAB4JwILAAAwPQILAAAwPc5hAQAApsc3LAAAwPQILAAAwPRy3e+wSH/dJ+H27dvy9fXN1p9ABwAA7mMYhlJSUpQvXz6nf8gwVwaW27dvKyYmxtNlAAAAF4SGhjr9c/65MrCkprLQ0FCX75TpKpvNppiYGI+s25Pom769AX3TtzfwZN+p63blNhG5MrCkHgayWCwe28g8uW5Pom/vQt/ehb69iyf7duV0Dk66BQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApud0YNm5c6f69OmjBg0ayGq1atOmTRlOO2rUKFmtVi1YsMBh+OXLl/Xaa68pIiJCtWrV0ogRI3Tt2jWniwcAAN7B6cCSlJQkq9Wq0aNH33W6b775Rnv37lVgYGCacYMHD9bhw4c1f/58zZo1S7t27dKoUaOcLQUAAHgJpwNLo0aN9Morr+jRRx/NcJrz589r7NixmjJlivLnz+8w7siRI9q6davGjRun6tWrq1atWho5cqTWrl2r8+fPO98BAADI89z+w3EpKSl6/fXX1bNnT/3rX/9KM3737t0qVqyYQkND7cMiIyPl6+urffv23TUI3clms7mlZmekrtMT6/Yk+qZvb0Df9O0NPNl3Vtbp9sAyZ84c5cuXT126dEl3fFxcnEqWLOlYRL588vf318WLF51alyfvJ+St9zKib+9C396Fvr1LbuvbrYFl//79WrRokVauXJkjd1HmXkI5h77p2xvQN317AzPcS8gVbg0su3btUnx8vJo0aWIfZrPZNGnSJC1atEibN29W6dKldenSJYf5bt++rStXriggIMCp9XEvoZxH396Fvr0LfXuX3Na3WwPLk08+qcjISIdhPXv21JNPPql27dpJksLDw5WQkKD9+/crJCREkrRjxw6lpKQoLCzMneUAAIA8wunAcu3aNZ08edL++PTp0zp48KD8/f1VtmxZlShRwmH6/Pnzq3Tp0qpUqZIkqXLlynrkkUf05ptvasyYMbp165bGjh2r6OholSlTJovt5Aw/Pz9PlwAAgFdxOrDs37/f4YTaCRMmSJLatm2riRMnZmoZU6ZM0dixY9W1a1f5+vrqscce08iRI50tJUtsKYYsvs6fZ2OxWBQcHOyRdQMA4K2cDix16tRRbGxspqffvHlzmmHFixfXe++95+yq3cri66NBS3fr8IXEHF1vlcAimvpseI6uEwCA3M7tlzXnJocvJOrA2QRPlwEAAO6Bmx8CAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTczqw7Ny5U3369FGDBg1ktVq1adMm+7hbt27p3XffVevWrVWjRg01aNBAQ4YM0fnz5x2WcfnyZb322muKiIhQrVq1NGLECF27di3r3QAAgDzJ6cCSlJQkq9Wq0aNHpxl3/fp1/fbbb+rbt69WrlypGTNm6NixY+rbt6/DdIMHD9bhw4c1f/58zZo1S7t27dKoUaNc7wIAAORp+ZydoVGjRmrUqFG644oWLar58+c7DHvzzTf1zDPP6OzZsypbtqyOHDmirVu36ssvv1RoaKgkaeTIkerdu7eGDBmiMmXKuNAGAADIy5wOLM5KTEyUj4+PihUrJknavXu3ihUrZg8rkhQZGSlfX1/t27dPjz76aKaXbbPZXK7LYrG4PK87ZKV2T0itN7fVnVX0Td/egL7pO6fX7YpsDSw3btzQlClTFB0drSJFikiS4uLiVLJkScci8uWTv7+/Ll686NTyY2JiXKrLz89PwcHBLs3rLrGxsUpOTvZoDa5w9TnP7ejbu9C3d6Hv3CHbAsutW7c0aNAgGYahMWPGZMs6QkNDPf5NiausVqunS3CKzWZTTExMrn7OXUHf9O0N6Ju+c3rdrsiWwHLr1i29/PLLOnv2rBYuXGj/dkWSSpcurUuXLjlMf/v2bV25ckUBAQFOrcdiseTajSw3151ba88K+vYu9O1d6Dt3cPvvsKSGlRMnTmjBggUqUaKEw/jw8HAlJCRo//799mE7duxQSkqKwsLC3F0OAADIA5z+huXatWs6efKk/fHp06d18OBB+fv7KyAgQAMHDtRvv/2m2bNny2az2c9L8ff3V4ECBVS5cmU98sgjevPNNzVmzBjdunVLY8eOVXR0NFcIAQCAdDkdWPbv368uXbrYH0+YMEGS1LZtW/Xv31+bN2+WJD355JMO8y1atEh16tSRJE2ZMkVjx45V165d5evrq8cee0wjR450uQkAAJC3OR1Y6tSpo9jY2AzH321cquLFi+u9995zdtUAAMBLcS8hAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgegQWAABgek4Hlp07d6pPnz5q0KCBrFarNm3a5DDeMAxNnTpVDRo0UFhYmLp166bjx487THP58mW99tprioiIUK1atTRixAhdu3YtS40AAIC8y+nAkpSUJKvVqtGjR6c7fs6cOVq8eLHeeustLV++XH5+furZs6du3Lhhn2bw4ME6fPiw5s+fr1mzZmnXrl0aNWqU610AAIA8zenA0qhRI73yyit69NFH04wzDEOLFi1S37591bx5c1WtWlWTJ0/WhQsX7N/EHDlyRFu3btW4ceNUvXp11apVSyNHjtTatWt1/vz5rHcEAADynHzuXNjp06d18eJFRUZG2ocVLVpU1atX1+7duxUdHa3du3erWLFiCg0NtU8TGRkpX19f7du3L90glBGbzeZyrRaLxeV53SErtXtCar25re6som/69gb0Td85vW5XuDWwXLx4UZJUqlQph+GlSpVSXFycJCkuLk4lS5Z0LCJfPvn7+9vnz6yYmBiX6vTz81NwcLBL87pLbGyskpOTPVqDK1x9znM7+vYu9O1d6Dt3cGtgyWmhoaEe/6bEVVar1dMlOMVmsykmJiZXP+euoG/69gb0Td85vW5XuDWwBAQESJLi4+MVGBhoHx4fH6+qVatKkkqXLq1Lly45zHf79m1duXLFPn9mWSyWXLuR5ea6c2vtWUHf3oW+vQt95w5u/R2W8uXLKyAgQNu3b7cPS0xM1N69exUeHi5JCg8PV0JCgvbv32+fZseOHUpJSVFYWJg7ywEAAHmE09+wXLt2TSdPnrQ/Pn36tA4ePCh/f3+VLVtWXbp00cyZM1WxYkWVL19eU6dOVWBgoJo3by5Jqly5sh555BG9+eabGjNmjG7duqWxY8cqOjpaZcqUcV9nAAAgz3A6sOzfv19dunSxP54wYYIkqW3btpo4caJ69eql5ORkjRo1SgkJCapZs6bmzp2rggUL2ueZMmWKxo4dq65du8rX11ePPfaYRo4c6YZ2AABAXuR0YKlTp45iY2MzHO/j46NBgwZp0KBBGU5TvHhxvffee86uGgAAeCnuJYRM8/Pz83QJAAAvRWDxMrYUw6X5LBaLgoODs3RGuavrBgAgV/8OC5xn8fXRoKW7dfhCYo6ut0pgEU19NjxH1wkAyDsILF7o8IVEHTib4OkyAADINA4JAQAA0yOwAPfAycYA4HkEFngFTjYGgNyNc1jgFTjZGAByNwILvAYnGwNA7sUhIQAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHpuDyw2m00ffPCBmjZtqrCwMDVv3lwffvihDMOwT2MYhqZOnaoGDRooLCxM3bp10/Hjx91dCgAAyCPcHljmzJmjJUuWaNSoUVq3bp0GDx6suXPnavHixQ7TLF68WG+99ZaWL18uPz8/9ezZUzdu3HB3OQBc5Ofn5+kSAMDO7YFl9+7datasmRo3bqzy5cvr8ccfV4MGDbRv3z5Jf327smjRIvXt21fNmzdX1apVNXnyZF24cEGbNm1ydzmAV7OlGPeeKB0Wi0XBwcGyWCw5vm4ASE8+dy8wPDxcy5cv17Fjx/TPf/5Tv//+u3755RcNGzZMknT69GldvHhRkZGR9nmKFi2q6tWra/fu3YqOjs70umw2m8t1ZmVH7A5ZqT0r6NszPNn3oKW7dfhCYo6ut0pgEU19Ntxjfbsqtd7cVndW0Td95/S6XeH2wNK7d28lJiaqZcuWslgsstlseuWVV/TEE09Iki5evChJKlWqlMN8pUqVUlxcnFPriomJcalGPz8/BQcHuzSvu8TGxio5OTlH10nfnuPJvg9fSNSBswk5uu5UnujbHVzdt+R29O1dclvfbg8s69ev15o1a/Tee++pSpUqOnjwoCZMmKDAwEC1bdvWresKDQ31+F/OrrJarZ4uwSPo27vktr5tNptiYmJy9b7FFfRN3zm9ble4PbBMnjxZvXv3th/asVqtOnv2rGbPnq22bdsqICBAkhQfH6/AwED7fPHx8apatapT67JYLLl2I8utdWcVfXuX3Np3bt63ZAV9e5fc1rfbT7q9fv26fHx8HIZZLBb7Zc3ly5dXQECAtm/fbh+fmJiovXv3Kjw83N3lAACAPMDt37A0adJEs2bNUtmyZe2HhObPn6+nnnpKkuTj46MuXbpo5syZqlixosqXL6+pU6cqMDBQzZs3d3c5AAAgD3B7YBk5cqSmTp2qMWPG2A/7dOjQQf369bNP06tXLyUnJ2vUqFFKSEhQzZo1NXfuXBUsWNDd5QAAgDzA7YGlSJEieuONN/TGG29kOI2Pj48GDRqkQYMGuXv1AAAgD+JeQgAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILADwN35+fp4uAUA6CCwA8hxbiuHSfBaLRcHBwbJYLDm+bgB3l8/TBQCAu1l8fTRo6W4dvpCYo+utElhEU58Nz9F1At6CwAIgTzp8IVEHziZ4ugwAbsIhIQAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAA91CC6RFYACCP4B5KyMuy5af5z58/r3fffVdbt25VcnKyKlasqPHjxys0NFSSZBiGpk2bpi+++EIJCQmKiIjQW2+9pQcffDA7ygEAr8A9lJCXuT2wXLlyRR07dlSdOnU0Z84clShRQidOnJC/v799mjlz5mjx4sWaOHGiypcvr6lTp6pnz55at26dChYs6O6SAMBrcA8l5FVuDyxz5szR/fffrwkTJtiHPfDAA/b/G4ahRYsWqW/fvmrevLkkafLkyYqMjNSmTZsUHR3t7pIAAEAu5/bAsnnzZjVo0EADBw7Uzp07VaZMGXXq1Ent27eXJJ0+fVoXL15UZGSkfZ6iRYuqevXq2r17t1OBxWazuVxnVo7VukNWas8K+vYM+s5Z9O0ZnurbVan15ra6s8qTfWdlnW4PLKdOndKSJUvUvXt39enTRzExMRo3bpzy58+vtm3b6uLFi5KkUqVKOcxXqlQpxcXFObWumJgYl2r08/NTcHCwS/O6S2xsrJKTk3N0nfTtOfSdc+jbczzRtzu4+lmS2+W2vt0eWAzDUEhIiF599VVJUnBwsA4dOqSlS5eqbdu2bl1XaGiox/+icJXVavV0CR5B396Fvr1LbuvbZrMpJiYmV3+WuMKTfaeu2xVuDywBAQGqXLmyw7BKlSrp66+/to+XpPj4eAUGBtqniY+PV9WqVZ1al8ViybUbWW6tO6vo27vQt3fJrX3n5s+SrMhtfbv9d1giIiJ07Ngxh2HHjx9XuXLlJEnly5dXQECAtm/fbh+fmJiovXv3Kjycy+IAAEBabg8sXbt21d69ezVr1iydOHFCa9as0fLly9WpUydJko+Pj7p06aKZM2fq22+/VWxsrIYMGaLAwED7VUMAAAB/5/ZDQmFhYZoxY4bef/99ffjhhypfvrxGjBihJ554wj5Nr169lJycrFGjRikhIUE1a9bU3Llz+Q0WAACQrmz5pdsmTZqoSZMmGY738fHRoEGDNGjQoOxYPQAAyGO4lxAAADA9AgsAADA9AgsAADA9AgsAwGv5+fl5ugRkEoEFAJCr2VIMl+azWCwKDg7O0o+nubpuOC9brhICACCnWHx9NGjpbh2+kJij660SWERTn+UHT3MKgQUAkOsdvpCoA2cTPF0GshGHhAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOlle2D5+OOPZbVa9c4779iH3bhxQ2PGjFGdOnUUHh6uAQMGKC4uLrtLAQAAuVS2BpZ9+/Zp6dKlslqtDsPHjx+v7777Th988IEWL16sCxcuqH///tlZCgAAyMWyLbBcu3ZNr7/+usaNGyd/f3/78KtXr2rFihUaNmyY6tWrp5CQEI0fP167d+/Wnj17sqscAACQi+XLrgW//fbbatSokSIjIzVz5kz78P379+vWrVuKjIy0D6tcubLKli2rPXv2qEaNGpleh81mc7k+i8Xi8rzukJXas4K+PYO+cxZ9ewZ95w6p9Xqi7qysM1sCy9q1a/Xbb7/pyy+/TDMuLi5O+fPnV7FixRyGlypVShcvXnRqPTExMS7V5+fnp+DgYJfmdZfY2FglJyfn6Drp23PoO+fQt+fQd+7i6meop7g9sJw7d07vvPOOPvnkExUsWNDdi3cQGhrq8WTtqjvP6/EW9O1d6Nu70HfuYLPZFBMT45HP0NR1u8LtgeXAgQOKj49Xu3bt7MNsNpt27typzz77TPPmzdOtW7eUkJDg8C1LfHy8AgICnFqXxWLJtYElt9adVfTtXejbu9B37pLbPkPdHljq1q2rNWvWOAwbPny4KlWqpF69eukf//iH8ufPr+3bt6tFixaSpKNHj+rs2bNOnb8CAAC8h9sDS5EiRRQUFOQwrFChQipevLh9+FNPPaWJEyfK399fRYoU0bhx4xQeHk5gAQAA6cq2q4TuZsSIEfL19dXAgQN18+ZNNWjQQKNHj/ZEKQAAIBfIkcCyePFih8cFCxbU6NGjCSkAACBTuJcQAAAwPQILAAAwPQILAAAwPQILAABexs/Pz9MlOI3AAgBALmRLMVyaz2KxKDg4OEs/GufqurPCI5c1AwCArLH4+mjQ0t06fCExR9dbJbCIpj4bnqPrlAgsAADkWocvJOrA2QRPl5EjOCQEAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMz+2BZfbs2XrqqacUHh6uevXq6aWXXtLRo0cdprlx44bGjBmjOnXqKDw8XAMGDFBcXJy7SwEAAHmE2wPLzz//rOeee07Lly/X/Pnzdfv2bfXs2VNJSUn2acaPH6/vvvtOH3zwgRYvXqwLFy6of//+7i4FAADkEfncvcB58+Y5PJ44caLq1aunAwcOqHbt2rp69apWrFihKVOmqF69epL+CjBRUVHas2ePatSo4e6SAABALuf2wHKnq1evSpL8/f0lSfv379etW7cUGRlpn6Zy5coqW7as04HFZrO5XJfFYnF5XnfISu1ZQd+eQd85i749g75zVm7sOyvPVbYGlpSUFI0fP14REREKCgqSJMXFxSl//vwqVqyYw7SlSpXSxYsXnVp+TEyMS3X5+fkpODjYpXndJTY2VsnJyTm6Tvr2HPrOOfTtOfSdc7yx72wNLGPGjNGhQ4f0+eefZ8vyQ0NDPZ4wXWW1Wj1dgkfQt3ehb+9C397Flb5tNpvLXzZkW2B5++239d///leffvqp7r//fvvw0qVL69atW0pISHD4liU+Pl4BAQFOrcNiseTawJJb684q+vYu9O1d6Nu75HTfbr9KyDAMvf322/rmm2+0cOFCPfDAAw7jQ0JClD9/fm3fvt0+7OjRozp79iwn3AIAgHS5/RuWMWPG6D//+Y8++ugjFS5c2H5eStGiRXXfffepaNGieuqppzRx4kT5+/urSJEiGjdunMLDwwksAAAgXW4PLEuWLJEkPf/88w7DJ0yYoHbt2kmSRowYIV9fXw0cOFA3b95UgwYNNHr0aHeXAgAA8gi3B5bY2Nh7TlOwYEGNHj2akAIAADKFewkBAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADT82hg+eyzz9S0aVOFhobqmWee0b59+zxZDgAAMCmPBZZ169ZpwoQJ6tevn1atWqWqVauqZ8+eio+P91RJAADApDwWWObPn6/27dvrqaeeUpUqVTRmzBjdd999WrFihadKAgAAJpXPEyu9efOmDhw4oBdffNE+zNfXV5GRkdq9e/c95zcMw74ci8XiUg0Wi0UP3V9YBV2b3WWVAgrLZrPJZrPl7Ir/f/Sds+ulb/rOSfSds+ulb+f7Tp0n9XPcGT6GK3Nl0fnz59WwYUMtXbpU4eHh9uGTJ0/Wzp079cUXX9x1/ps3byomJia7ywQAANkgNDRUBQoUcGoej3zDklX58uVTaGiofH195ePj4+lyAABAJhiGoZSUFOXL53z88EhgKVGihCwWS5oTbOPj41W6dOl7zu/r6+t0MgMAALmXR066LVCggKpVq6bt27fbh6WkpGj79u0Oh4gAAAAkDx4S6t69u4YOHaqQkBCFhYVp4cKFSk5OVrt27TxVEgAAMCmPBZaoqChdunRJ06ZN08WLF/XQQw9p7ty5mTokBAAAvItHrhICAABwBvcSAgAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAAppcr7yUEAHCPmzdvShK3O8nDbt68qU2bNmnPnj2Ki4uTJJUuXVrh4eFq1qxZrnnt+R0WAJKkS5cuacWKFenu1Nq1a6eSJUt6uEL3O3z4sD799NM0PdeoUUOdO3dWlSpVPFxh9ti2bZsWLFigPXv2KDExUZJUpEgR1ahRQ927d1dkZKSHK4S7nDhxQj179tSFCxdUvXp1lSpVStJf9+7bu3ev7r//fs2ZM0cVK1b0cKX3RmC5B2/doUne2bs39ixJ+/bt0wsvvKD77rtPkZGRDju17du36/r165o7d65CQ0M9XKn7bNmyRf369VO1atXUoEEDh563bdumAwcO6KOPPtIjjzzi4Urda9WqVRo5cqRatGiRbt9ff/21xo0bpzZt2ni20Gzgje/v7t27y8/PT5MnT1aRIkUcxiUmJmrIkCG6ceOG5s2b56EKM4/AchfeukOTvLN3b+w5Vfv27VW1alWNGTNGPj4+DuMMw9Do0aMVGxurZcuWeahC93viiSfUrFkzDRo0KN3x06dP18aNG7VmzZocrix7tWjRQl26dNFzzz2X7vjPPvtMCxcu1MaNG3O4suzlre/v6tWr64svvlBQUFC642NjY9W+fXvt3bs3hytzgYEMtW7d2vjggw8yHD9t2jSjVatWOVhRzvHG3r2x51ShoaHG4cOHMxx/+PBhIzQ0NAcryn6hoaHGkSNHMhx/5MiRPNezYRhGSEiIV/btre/v+vXrG5s3b85w/LfffmvUr18/BytyHVcJ3cXx48fVunXrDMdHR0frxIkTOVhRzvHG3r2x51SlS5dWTExMhuNjYmLy3I1Jy5Urpy1btmQ4fsuWLSpbtmwOVpQz/vWvf+nLL7/McPyKFSvy5KERb31/P/PMMxo6dKgWLFig33//XXFxcYqLi9Pvv/+uBQsWaPjw4erQoYOny8wUrhK6i9QdWqVKldIdn1d3aJJ39u6NPafq2bOn3nzzTe3fv1/16tWzh5O4uDht375dX3zxhYYMGeLhKt1r4MCBGjx4sH766SdFRkam6Xnr1q167733PFyl+w0dOlR9+vTR1q1b0z1f6dSpU/r44489XKX7eev7e9CgQfLz89PcuXM1ceJE+yFfwzBUunRpvfDCC+rVq5eHq8wczmG5i/Xr12vw4MF65JFH7rpDa9GihYcrdT9v7N0be/67devWacGCBTpw4IBsNpskyWKxqFq1aurWrZuioqI8XKH7/frrr1q8eLH27NmjixcvSpICAgJUo0YNdenSReHh4R6uMHucPn1aS5Ys0d69e9P0/eyzz6p8+fIertD9vP39LUmnTp1yONn4gQce8HBFziGw3IO37tAk7+zdG3u+061bt/Tnn39KkkqUKKH8+fN7uCLAPXh/524EFgAAvNSmTZuUmJiYKy5jJ7AAuKfPPvtMf/75p/r37+/pUnLM+++/r4sXL2rChAmeLiVHDR06VOfOndOiRYs8XQpywOOPP64TJ07o4MGDni7lnrhKKAvef/99DR8+3NNleIQ39u6NPafauHGjVq1a5ekyctT58+d15swZT5eR48qUKaNy5cp5uowc563v7w0bNuSKsCJxlVCW/PHHH/rjjz88XYZHnD9/XufOnfN0GTnKm1/vhQsXerqEHGMYhnx8fDRp0iRPl+IRr776qqdL8Ahv3KflNhwSAoC/CQkJ0f/93/+pcuXKni4FcJt9+/Zp9+7dae4TFhYW5uHKMo/Acg9HjhzRnj17VKNGDVWuXFlHjhzRokWLdPPmTT3xxBOqV6+ep0tENklKStL69et18uRJBQQEKDo6WiVKlPB0Wdkmr9zRNbMyOjdl0aJFeuKJJ1S8eHFJynOHCQ4cOKBixYrZL2ldvXq1li5dqnPnzqls2bLq3LmzoqOjPVyl+40dO1YtW7ZUrVq1PF1KjoqPj9eAAQP066+/qmzZsg6/u3P27FlFRERo+vTp9uFmRmC5i++//14vvfSSChcurOTkZM2YMUNDhw5V1apVlZKSop07d2revHleGVri4uK0dOnSPHUSZlRUlD7//HMVL15c586d03PPPaeEhAQ9+OCDOnXqlCwWi5YtW5brfrsgM/LSHV0zq2rVqqpataqKFi3qMHznzp0KCQmRn5+ffHx88tzJp0888YSGDRumyMhIffHFFxo3bpyeeeYZVa5cWceOHdMXX3yhN954Q08//bSnS3WrqlWrysfHRxUqVNBTTz2ltm3bKiAgwNNlZbuBAwfqwoULGj9+fJofzTt69KhGjBihwMBATZs2zUMVOiHn7waQe3To0MF4//33DcMwjP/85z9G7dq17Y8NwzCmTJlidO/e3VPledTBgweNqlWreroMt7JarUZcXJxhGIbx2muvGR06dDASEhIMwzCMxMREo1u3bsarr77qyRKzTbdu3Yy+ffsaV69eTTPu6tWrRt++fY0ePXp4oLLsM3v2bKNp06bGjz/+6DA8ODjYOHTokIeqyn5hYWHG6dOnDcMwjDZt2hjLli1zGP/VV18ZUVFRnigtW1mtVuPHH380xo0bZ9SpU8eoVq2a0adPH2Pz5s2GzWbzdHnZpkaNGsaBAwcyHB8TE2PUqFEjBytyHVcJ3cWhQ4fUrl07SVLLli117do1h19BbN26tWJjYz1VXrb6/fff7/rv6NGjni4xW+3Zs0cDBgyw//VduHBhDRgwQL/88ouHK8sev/76q15++eU0t5+XpCJFimjQoEHatWuXByrLPr1799a///1vvfXWW5o0aZJu3brl6ZJyxH333Wf/YcDz58+nOYehevXqOn36tCdKy3ZBQUF64403tHXrVk2ePFk3b95Uv3791KhRI/373//Ok/cSKlCggBITEzMcf+3atVxzuJerhO4h9b4Lvr6+KlCggMPXx4ULF9bVq1c9VVq2atOmjXx8fGSkc8QwdXjqc5OXpPZ048aNNF8XlylTRpcuXfJEWdmuaNGiOnPmTIa3oD9z5kyaQyd5QVhYmFauXKm3335bTz31lKZMmZInt+u/a9iwoZYsWaJ33nlHtWvX1oYNG1S1alX7+PXr16tChQoerDD75c+fX1FRUYqKitLZs2e1YsUKrVq1Sh9//HGuucQ3s6KiojRs2DANHz5c9erVs/9RkpiYqO3bt2vChAlq1aqVh6vMHALLXZQrV07Hjx+3v3mXLVumf/zjH/bx586dy7PHQP39/fX6669neH7O4cOH1adPnxyuKvt17dpV+fLlU2Jioo4dO+bwAX727Fn7iZh5TeodXV966SXVrVvX4T4rO3bs0MyZM9W5c2cPV5k9ChcurEmTJmnt2rXq3r27/T5KedXgwYPVsWNHde7cWSEhIZo/f75+/vln+zkse/bs0YcffujpMnNM2bJlNWDAAPXv318//vijp8txu+HDhyslJUWvvPKKbDab/VYbt27dksVi0dNPP62hQ4d6uMrMIbDcRceOHZWSkmJ/fOdfn99//73q1q2b02XliJCQEF24cCHDH5C6evVqut++5GZ3nkBcqFAhh8ebN2/Os1cY5KU7uroqOjpaNWvW1P79+/PkXXtTlSlTRqtXr9bHH3+s7777ToZhaN++ffrjjz8UHh6uJUuWKDQ01NNlul3ZsmXl65vxWRA+Pj6qX79+DlaUMwoUKKAxY8bo9ddf1/79+x2uAAwJCUn3MLBZcZUQ0vXNN98oKSlJTz75ZLrjr1y5os2bN6tt27Y5XBmyW26/oyuAvInAAuCezp07p2nTpnndfXWAvOD69evav3+/ihcvripVqjiMu3HjhtavX58rbn7IVUJwyblz5/LcD2ohY1euXNHq1as9XQYAJx07dkxRUVHq3LmzWrdurc6dO+v8+fP28VevXs01+3LOYYFLUj/A+Is7b/j222/vOv7UqVM5VAkAd5oyZYr+9a9/6csvv9TVq1c1fvx4derUSYsXL85152oRWJAuPsC8S79+/TK8jD1VXr/cF8iLdu/erfnz56tkyZIqWbKkZs2apbfeekvPPfecFi1aJD8/P0+XmGkEFqSLDzDvEhAQoNGjR6t58+bpjj948KD9RxQB5B7Xr19Xvnz/76Pex8dHY8aM0dtvv63OnTvrvffe82B1ziGwIF18gHmXatWq6cCBAxm+3vcKrwDMqVKlSoqJiUlz9/FRo0ZJkvr27euJslzCSbdIV+oHWEb4AMtbXnjhBYWHh2c4vkKFCnnuJoCAN3j00Ue1du3adMeNGjVK0dHRuWZfzmXNSNeuXbuUlJSkhg0bpjs+KSlJ+/fv18MPP5zDlQEAvBGBBQAAmB6HhAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOn9fyAO4c+WCfHFAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# output information about selected dataset\n",
        "datasets_df.loc[40900]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TNlbl7Rpsx-Q",
        "outputId": "871f5083-adeb-4d79-88bd-fd66721ed24e"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "name                                  Satellite\n",
              "version                                       1\n",
              "uploader                                   3768\n",
              "status                                   active\n",
              "format                                     ARFF\n",
              "MajorityClassSize                        5025.0\n",
              "MaxNominalAttDistinctValues                 2.0\n",
              "MinorityClassSize                          75.0\n",
              "NumberOfClasses                             2.0\n",
              "NumberOfFeatures                           37.0\n",
              "NumberOfInstances                        5100.0\n",
              "NumberOfInstancesWithMissingValues          0.0\n",
              "NumberOfMissingValues                       0.0\n",
              "NumberOfNumericFeatures                    36.0\n",
              "NumberOfSymbolicFeatures                    1.0\n",
              "Name: 40900, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#  Satellite dataset https://www.openml.org/d/182\n",
        "dataset = openml.datasets.get_dataset( 182)\n",
        "\n",
        "# Print a summary\n",
        "print(\n",
        "    f\"This is dataset '{dataset.name}', the target feature is \"\n",
        "    f\"'{dataset.default_target_attribute}'\"\n",
        ")\n",
        "print(f\"URL: {dataset.url}\")\n",
        "print(dataset.description[:500])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sZKMX2uRs4Fn",
        "outputId": "cf47084b-65bc-41d5-8e2a-642a2492c3ec"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "This is dataset 'satimage', the target feature is 'class'\n",
            "URL: https://api.openml.org/data/v1/download/3619/satimage.arff\n",
            "**Author**: Ashwin Srinivasan, Department of Statistics and Data Modeling, University of Strathclyde  \n",
            "**Source**: [UCI](https://archive.ics.uci.edu/ml/datasets/Statlog+(Landsat+Satellite)) - 1993  \n",
            "**Please cite**: [UCI](https://archive.ics.uci.edu/ml/citation_policy.html)  \n",
            "\n",
            "The database consists of the multi-spectral values of pixels in 3x3 neighbourhoods in a satellite image, and the classification associated with the central pixel in each neighbourhood. The aim is to predict this classifica\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# openml API\n",
        "X, y, categorical_indicator, attribute_names = dataset.get_data(\n",
        "    dataset_format=\"array\", target=dataset.default_target_attribute\n",
        ")\n",
        "df = pd.DataFrame(X, columns=attribute_names)\n",
        "df[\"class\"] = y"
      ],
      "metadata": {
        "id": "87Yz-u-6tSYO"
      },
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        },
        "id": "X4eRAC-htZbK",
        "outputId": "e5e942fa-f4b3-4413-f5ef-de20ea497b2b"
      },
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "      Aattr     Battr     Cattr     Dattr     Eattr     Fattr    A1attr  \\\n",
              "0  0.117596  1.241362  1.184036  0.815302 -0.158561  1.256483  1.193546   \n",
              "1 -1.205362 -1.249654 -0.077532  0.444886 -0.895959 -0.447579 -0.786760   \n",
              "2  0.779075  0.148811  0.042617 -0.243030  0.800057  0.164136  0.053370   \n",
              "3  1.146564  0.585831  0.342991  0.021553  0.947536  0.601074  0.353416   \n",
              "4 -0.764376 -1.162250 -0.137607  0.180303 -0.969698 -1.146681 -0.126658   \n",
              "\n",
              "     B2attr    C3attr    D4attr    E5attr    F6attr    A7attr    B8attr  \\\n",
              "0  0.818486 -0.141965  0.879481  0.670010  0.401020  0.052220  1.204523   \n",
              "1 -0.554203 -0.364672  0.092157 -0.051291 -0.178503 -0.976988 -0.852379   \n",
              "2 -0.448612  0.154978 -0.345245 -0.712483 -0.441923  0.713854  0.504301   \n",
              "3  0.026550  1.788164  1.010702  0.910444  0.401020  1.375487  0.810648   \n",
              "4  0.184937 -0.735851 -1.132569 -0.111399  0.190284 -1.197532 -1.202490   \n",
              "\n",
              "     C9attr   D10attr   E11attr   F12attr   A13attr   B14attr   C15attr  \\\n",
              "0  1.181239  0.758245 -0.151111  1.214967  1.187378  0.598708 -0.136658   \n",
              "1 -0.498640  0.124397 -0.668189 -0.836981 -0.487496 -0.084225 -0.656748   \n",
              "2 -0.198662 -0.403809  0.735308  0.341798 -0.188412 -0.399425  0.160536   \n",
              "3  0.941256  0.230039  1.104649  1.214967  0.409758  0.230975  1.126416   \n",
              "4 -0.138666  0.388501 -0.668189 -1.186248 -0.367863  0.230975 -0.879643   \n",
              "\n",
              "    D16attr   E17attr   F18attr   A19attr   B20attr   C21attr   D22attr  \\\n",
              "0  1.011130  0.899623  0.761977 -0.085593  1.211546  1.251179  0.807707   \n",
              "1 -0.343506 -0.176313 -0.234313 -0.748307 -0.981173 -0.614884 -0.192752   \n",
              "2 -0.343506 -0.355636 -0.181877  0.650756 -0.016376 -0.915862 -0.877277   \n",
              "3  1.011130  0.720300  0.237614  1.387105  0.860711  0.528832  0.281150   \n",
              "4 -0.998975 -0.355636 -0.129441 -0.895577 -1.244299 -0.795470 -0.192752   \n",
              "\n",
              "    E23attr   F24attr   A25attr   B26attr   C27attr   D28attr   E29attr  \\\n",
              "0 -0.069968  1.219160  1.250463  0.597678 -0.054291  1.233342  1.262255   \n",
              "1 -0.736996 -0.969292 -0.844805 -0.400030 -0.725852 -0.344432 -0.594534   \n",
              "2  0.671174 -0.006373 -0.425752 -0.662584  0.691889  0.356801 -0.175259   \n",
              "3  1.412317  1.044084  0.532085  0.282612  1.438068  1.058033  0.842981   \n",
              "4 -0.885225 -1.231906 -0.784941 -0.347519 -0.875088 -1.220973 -0.774223   \n",
              "\n",
              "    F30attr  class  \n",
              "0  0.603258      0  \n",
              "1 -0.183967      4  \n",
              "2 -0.236449      5  \n",
              "3  0.130923      2  \n",
              "4 -0.551339      4  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-1c897798-f0c5-4570-84d1-5e41c712b111\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Aattr</th>\n",
              "      <th>Battr</th>\n",
              "      <th>Cattr</th>\n",
              "      <th>Dattr</th>\n",
              "      <th>Eattr</th>\n",
              "      <th>Fattr</th>\n",
              "      <th>A1attr</th>\n",
              "      <th>B2attr</th>\n",
              "      <th>C3attr</th>\n",
              "      <th>D4attr</th>\n",
              "      <th>E5attr</th>\n",
              "      <th>F6attr</th>\n",
              "      <th>A7attr</th>\n",
              "      <th>B8attr</th>\n",
              "      <th>C9attr</th>\n",
              "      <th>D10attr</th>\n",
              "      <th>E11attr</th>\n",
              "      <th>F12attr</th>\n",
              "      <th>A13attr</th>\n",
              "      <th>B14attr</th>\n",
              "      <th>C15attr</th>\n",
              "      <th>D16attr</th>\n",
              "      <th>E17attr</th>\n",
              "      <th>F18attr</th>\n",
              "      <th>A19attr</th>\n",
              "      <th>B20attr</th>\n",
              "      <th>C21attr</th>\n",
              "      <th>D22attr</th>\n",
              "      <th>E23attr</th>\n",
              "      <th>F24attr</th>\n",
              "      <th>A25attr</th>\n",
              "      <th>B26attr</th>\n",
              "      <th>C27attr</th>\n",
              "      <th>D28attr</th>\n",
              "      <th>E29attr</th>\n",
              "      <th>F30attr</th>\n",
              "      <th>class</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.117596</td>\n",
              "      <td>1.241362</td>\n",
              "      <td>1.184036</td>\n",
              "      <td>0.815302</td>\n",
              "      <td>-0.158561</td>\n",
              "      <td>1.256483</td>\n",
              "      <td>1.193546</td>\n",
              "      <td>0.818486</td>\n",
              "      <td>-0.141965</td>\n",
              "      <td>0.879481</td>\n",
              "      <td>0.670010</td>\n",
              "      <td>0.401020</td>\n",
              "      <td>0.052220</td>\n",
              "      <td>1.204523</td>\n",
              "      <td>1.181239</td>\n",
              "      <td>0.758245</td>\n",
              "      <td>-0.151111</td>\n",
              "      <td>1.214967</td>\n",
              "      <td>1.187378</td>\n",
              "      <td>0.598708</td>\n",
              "      <td>-0.136658</td>\n",
              "      <td>1.011130</td>\n",
              "      <td>0.899623</td>\n",
              "      <td>0.761977</td>\n",
              "      <td>-0.085593</td>\n",
              "      <td>1.211546</td>\n",
              "      <td>1.251179</td>\n",
              "      <td>0.807707</td>\n",
              "      <td>-0.069968</td>\n",
              "      <td>1.219160</td>\n",
              "      <td>1.250463</td>\n",
              "      <td>0.597678</td>\n",
              "      <td>-0.054291</td>\n",
              "      <td>1.233342</td>\n",
              "      <td>1.262255</td>\n",
              "      <td>0.603258</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>-1.205362</td>\n",
              "      <td>-1.249654</td>\n",
              "      <td>-0.077532</td>\n",
              "      <td>0.444886</td>\n",
              "      <td>-0.895959</td>\n",
              "      <td>-0.447579</td>\n",
              "      <td>-0.786760</td>\n",
              "      <td>-0.554203</td>\n",
              "      <td>-0.364672</td>\n",
              "      <td>0.092157</td>\n",
              "      <td>-0.051291</td>\n",
              "      <td>-0.178503</td>\n",
              "      <td>-0.976988</td>\n",
              "      <td>-0.852379</td>\n",
              "      <td>-0.498640</td>\n",
              "      <td>0.124397</td>\n",
              "      <td>-0.668189</td>\n",
              "      <td>-0.836981</td>\n",
              "      <td>-0.487496</td>\n",
              "      <td>-0.084225</td>\n",
              "      <td>-0.656748</td>\n",
              "      <td>-0.343506</td>\n",
              "      <td>-0.176313</td>\n",
              "      <td>-0.234313</td>\n",
              "      <td>-0.748307</td>\n",
              "      <td>-0.981173</td>\n",
              "      <td>-0.614884</td>\n",
              "      <td>-0.192752</td>\n",
              "      <td>-0.736996</td>\n",
              "      <td>-0.969292</td>\n",
              "      <td>-0.844805</td>\n",
              "      <td>-0.400030</td>\n",
              "      <td>-0.725852</td>\n",
              "      <td>-0.344432</td>\n",
              "      <td>-0.594534</td>\n",
              "      <td>-0.183967</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.779075</td>\n",
              "      <td>0.148811</td>\n",
              "      <td>0.042617</td>\n",
              "      <td>-0.243030</td>\n",
              "      <td>0.800057</td>\n",
              "      <td>0.164136</td>\n",
              "      <td>0.053370</td>\n",
              "      <td>-0.448612</td>\n",
              "      <td>0.154978</td>\n",
              "      <td>-0.345245</td>\n",
              "      <td>-0.712483</td>\n",
              "      <td>-0.441923</td>\n",
              "      <td>0.713854</td>\n",
              "      <td>0.504301</td>\n",
              "      <td>-0.198662</td>\n",
              "      <td>-0.403809</td>\n",
              "      <td>0.735308</td>\n",
              "      <td>0.341798</td>\n",
              "      <td>-0.188412</td>\n",
              "      <td>-0.399425</td>\n",
              "      <td>0.160536</td>\n",
              "      <td>-0.343506</td>\n",
              "      <td>-0.355636</td>\n",
              "      <td>-0.181877</td>\n",
              "      <td>0.650756</td>\n",
              "      <td>-0.016376</td>\n",
              "      <td>-0.915862</td>\n",
              "      <td>-0.877277</td>\n",
              "      <td>0.671174</td>\n",
              "      <td>-0.006373</td>\n",
              "      <td>-0.425752</td>\n",
              "      <td>-0.662584</td>\n",
              "      <td>0.691889</td>\n",
              "      <td>0.356801</td>\n",
              "      <td>-0.175259</td>\n",
              "      <td>-0.236449</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.146564</td>\n",
              "      <td>0.585831</td>\n",
              "      <td>0.342991</td>\n",
              "      <td>0.021553</td>\n",
              "      <td>0.947536</td>\n",
              "      <td>0.601074</td>\n",
              "      <td>0.353416</td>\n",
              "      <td>0.026550</td>\n",
              "      <td>1.788164</td>\n",
              "      <td>1.010702</td>\n",
              "      <td>0.910444</td>\n",
              "      <td>0.401020</td>\n",
              "      <td>1.375487</td>\n",
              "      <td>0.810648</td>\n",
              "      <td>0.941256</td>\n",
              "      <td>0.230039</td>\n",
              "      <td>1.104649</td>\n",
              "      <td>1.214967</td>\n",
              "      <td>0.409758</td>\n",
              "      <td>0.230975</td>\n",
              "      <td>1.126416</td>\n",
              "      <td>1.011130</td>\n",
              "      <td>0.720300</td>\n",
              "      <td>0.237614</td>\n",
              "      <td>1.387105</td>\n",
              "      <td>0.860711</td>\n",
              "      <td>0.528832</td>\n",
              "      <td>0.281150</td>\n",
              "      <td>1.412317</td>\n",
              "      <td>1.044084</td>\n",
              "      <td>0.532085</td>\n",
              "      <td>0.282612</td>\n",
              "      <td>1.438068</td>\n",
              "      <td>1.058033</td>\n",
              "      <td>0.842981</td>\n",
              "      <td>0.130923</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>-0.764376</td>\n",
              "      <td>-1.162250</td>\n",
              "      <td>-0.137607</td>\n",
              "      <td>0.180303</td>\n",
              "      <td>-0.969698</td>\n",
              "      <td>-1.146681</td>\n",
              "      <td>-0.126658</td>\n",
              "      <td>0.184937</td>\n",
              "      <td>-0.735851</td>\n",
              "      <td>-1.132569</td>\n",
              "      <td>-0.111399</td>\n",
              "      <td>0.190284</td>\n",
              "      <td>-1.197532</td>\n",
              "      <td>-1.202490</td>\n",
              "      <td>-0.138666</td>\n",
              "      <td>0.388501</td>\n",
              "      <td>-0.668189</td>\n",
              "      <td>-1.186248</td>\n",
              "      <td>-0.367863</td>\n",
              "      <td>0.230975</td>\n",
              "      <td>-0.879643</td>\n",
              "      <td>-0.998975</td>\n",
              "      <td>-0.355636</td>\n",
              "      <td>-0.129441</td>\n",
              "      <td>-0.895577</td>\n",
              "      <td>-1.244299</td>\n",
              "      <td>-0.795470</td>\n",
              "      <td>-0.192752</td>\n",
              "      <td>-0.885225</td>\n",
              "      <td>-1.231906</td>\n",
              "      <td>-0.784941</td>\n",
              "      <td>-0.347519</td>\n",
              "      <td>-0.875088</td>\n",
              "      <td>-1.220973</td>\n",
              "      <td>-0.774223</td>\n",
              "      <td>-0.551339</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1c897798-f0c5-4570-84d1-5e41c712b111')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-1c897798-f0c5-4570-84d1-5e41c712b111 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-1c897798-f0c5-4570-84d1-5e41c712b111');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-922d8853-e1c2-49d2-8b4a-665aa63bb9bc\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-922d8853-e1c2-49d2-8b4a-665aa63bb9bc')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-922d8853-e1c2-49d2-8b4a-665aa63bb9bc button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['class'].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CS-Vb6dUtbhT",
        "outputId": "ab303b65-13f9-4246-98cf-6c7e21e2c137"
      },
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    1531\n",
              "5    1508\n",
              "2    1356\n",
              "4     707\n",
              "1     703\n",
              "3     625\n",
              "Name: class, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.describe()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 317
        },
        "id": "tSSTkok-twBu",
        "outputId": "af0a18ef-d8e4-45e8-aa81-b78e510ad1b8"
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "             Aattr        Battr        Cattr        Dattr        Eattr  \\\n",
              "count  6430.000000  6430.000000  6430.000000  6430.000000  6430.000000   \n",
              "mean     -0.000846    -0.000748    -0.000603    -0.000272    -0.000863   \n",
              "std       0.999649     0.999944     0.999965     1.000196     0.999696   \n",
              "min      -2.234329    -2.473310    -2.780894    -2.624275    -2.223275   \n",
              "25%      -0.690878    -0.550421    -0.858503    -0.719279    -0.674739   \n",
              "50%      -0.102897     0.148811     0.102692    -0.084280    -0.084821   \n",
              "75%       0.779075     0.848043     0.823588     0.497802     0.800057   \n",
              "max       2.543020     2.333912     2.445605     3.778629     2.569812   \n",
              "\n",
              "             Fattr       A1attr       B2attr       C3attr       D4attr  \\\n",
              "count  6430.000000  6430.000000  6430.000000  6430.000000  6430.000000   \n",
              "mean     -0.000787    -0.000611    -0.000333    -0.000767    -0.000779   \n",
              "std       0.999958     0.999968     1.000220     0.999807     0.999904   \n",
              "min      -2.457498    -2.947095    -2.824420    -2.146330    -2.444776   \n",
              "25%      -0.534967    -0.846770    -0.712591    -0.661615    -0.520206   \n",
              "50%       0.076748     0.113379    -0.079042    -0.141965     0.092157   \n",
              "75%       0.863238     0.833490     0.501711     0.748864     0.835741   \n",
              "max       2.348830     2.753787     3.933434     2.604758     2.060467   \n",
              "\n",
              "            E5attr       F6attr       A7attr       B8attr       C9attr  \\\n",
              "count  6430.000000  6430.000000  6430.000000  6430.000000  6430.000000   \n",
              "mean     -0.000540    -0.000279    -0.000829    -0.000760    -0.000536   \n",
              "std       0.999959     1.000203     0.999639     0.999957     1.000093   \n",
              "min      -2.936494    -2.812701    -2.226740    -2.471642    -2.958463   \n",
              "25%      -0.832700    -0.758027    -0.682928    -0.546032    -0.858614   \n",
              "50%       0.068926    -0.073135    -0.094810     0.066663     0.101317   \n",
              "75%       0.850335     0.506388     0.787369     0.854412     0.821265   \n",
              "max       2.773804     3.930845     2.551725     2.342384     2.741127   \n",
              "\n",
              "           D10attr      E11attr      F12attr      A13attr      B14attr  \\\n",
              "count  6430.000000  6430.000000  6430.000000  6430.000000  6430.000000   \n",
              "mean     -0.000294    -0.000698    -0.000768    -0.000467    -0.000204   \n",
              "std       1.000232     0.999876     0.999964     1.000127     1.000264   \n",
              "min      -2.833559    -2.145554    -2.452344    -2.939991    -2.815957   \n",
              "25%      -0.720733    -0.668189    -0.531371    -0.846398    -0.714624   \n",
              "50%      -0.086885    -0.077243     0.079847     0.050856    -0.084225   \n",
              "75%       0.494142     0.735308     0.865700     0.828476     0.493642   \n",
              "max       3.769022     2.582013     2.044478     2.742619     3.908307   \n",
              "\n",
              "           C15attr      D16attr      E17attr      F18attr      A19attr  \\\n",
              "count  6430.000000  6430.000000  6430.000000  6430.000000  6430.000000   \n",
              "mean     -0.000599    -0.000732    -0.000579    -0.000274    -0.000598   \n",
              "std       1.000002     0.999981     1.000074     1.000218     1.000037   \n",
              "min      -2.217016    -2.441006    -2.925927    -2.803693    -2.221005   \n",
              "25%      -0.656748    -0.518297    -0.833829    -0.758677    -0.674672   \n",
              "50%      -0.136658     0.049776     0.062784    -0.077004    -0.085593   \n",
              "75%       0.754923     0.880036     0.839849     0.499795     0.724391   \n",
              "max       2.612386     2.059880     2.752623     3.908156     2.565264   \n",
              "\n",
              "           B20attr      C21attr      D22attr      E23attr      F24attr  \\\n",
              "count  6430.000000  6430.000000  6430.000000  6430.000000  6430.000000   \n",
              "mean     -0.000710    -0.000430    -0.000178    -0.000626    -0.000717   \n",
              "std       1.000000     1.000192     1.000271     0.999897     1.000022   \n",
              "min      -2.472222    -2.962511    -2.825539    -2.219281    -2.457440   \n",
              "25%      -0.542629    -0.855666    -0.719309    -0.662882    -0.531602   \n",
              "50%       0.071332     0.047268    -0.087440    -0.069968     0.081165   \n",
              "75%       0.860711     0.829810     0.491773     0.745289     0.869008   \n",
              "max       2.088634     2.455091     3.756429     2.598145     2.050772   \n",
              "\n",
              "           A25attr      B26attr      C27attr      D28attr      E29attr  \\\n",
              "count  6430.000000  6430.000000  6430.000000  6430.000000  6430.000000   \n",
              "mean     -0.000525    -0.000212    -0.000596    -0.000666    -0.000469   \n",
              "std       1.000017     1.000271     0.999878     1.000013     1.000009   \n",
              "min      -2.940074    -2.815531    -2.218211    -2.448131    -2.930494   \n",
              "25%      -0.844805    -0.715095    -0.651234    -0.519740    -0.834119   \n",
              "50%       0.053167    -0.084964    -0.128909     0.050011     0.064327   \n",
              "75%       0.831409     0.492656     0.766507     0.838898     0.842981   \n",
              "max       2.747083     3.905865     2.631955     2.066056     2.759666   \n",
              "\n",
              "           F30attr        class  \n",
              "count  6430.000000  6430.000000  \n",
              "mean     -0.000126     2.435148  \n",
              "std       1.000288     1.876348  \n",
              "min      -2.808050     0.000000  \n",
              "25%      -0.761265     1.000000  \n",
              "50%      -0.079004     2.000000  \n",
              "75%       0.498294     4.000000  \n",
              "max       3.909603     5.000000  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8994e2aa-366d-4d81-b749-c8269e5a87c3\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Aattr</th>\n",
              "      <th>Battr</th>\n",
              "      <th>Cattr</th>\n",
              "      <th>Dattr</th>\n",
              "      <th>Eattr</th>\n",
              "      <th>Fattr</th>\n",
              "      <th>A1attr</th>\n",
              "      <th>B2attr</th>\n",
              "      <th>C3attr</th>\n",
              "      <th>D4attr</th>\n",
              "      <th>E5attr</th>\n",
              "      <th>F6attr</th>\n",
              "      <th>A7attr</th>\n",
              "      <th>B8attr</th>\n",
              "      <th>C9attr</th>\n",
              "      <th>D10attr</th>\n",
              "      <th>E11attr</th>\n",
              "      <th>F12attr</th>\n",
              "      <th>A13attr</th>\n",
              "      <th>B14attr</th>\n",
              "      <th>C15attr</th>\n",
              "      <th>D16attr</th>\n",
              "      <th>E17attr</th>\n",
              "      <th>F18attr</th>\n",
              "      <th>A19attr</th>\n",
              "      <th>B20attr</th>\n",
              "      <th>C21attr</th>\n",
              "      <th>D22attr</th>\n",
              "      <th>E23attr</th>\n",
              "      <th>F24attr</th>\n",
              "      <th>A25attr</th>\n",
              "      <th>B26attr</th>\n",
              "      <th>C27attr</th>\n",
              "      <th>D28attr</th>\n",
              "      <th>E29attr</th>\n",
              "      <th>F30attr</th>\n",
              "      <th>class</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "      <td>6430.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>-0.000846</td>\n",
              "      <td>-0.000748</td>\n",
              "      <td>-0.000603</td>\n",
              "      <td>-0.000272</td>\n",
              "      <td>-0.000863</td>\n",
              "      <td>-0.000787</td>\n",
              "      <td>-0.000611</td>\n",
              "      <td>-0.000333</td>\n",
              "      <td>-0.000767</td>\n",
              "      <td>-0.000779</td>\n",
              "      <td>-0.000540</td>\n",
              "      <td>-0.000279</td>\n",
              "      <td>-0.000829</td>\n",
              "      <td>-0.000760</td>\n",
              "      <td>-0.000536</td>\n",
              "      <td>-0.000294</td>\n",
              "      <td>-0.000698</td>\n",
              "      <td>-0.000768</td>\n",
              "      <td>-0.000467</td>\n",
              "      <td>-0.000204</td>\n",
              "      <td>-0.000599</td>\n",
              "      <td>-0.000732</td>\n",
              "      <td>-0.000579</td>\n",
              "      <td>-0.000274</td>\n",
              "      <td>-0.000598</td>\n",
              "      <td>-0.000710</td>\n",
              "      <td>-0.000430</td>\n",
              "      <td>-0.000178</td>\n",
              "      <td>-0.000626</td>\n",
              "      <td>-0.000717</td>\n",
              "      <td>-0.000525</td>\n",
              "      <td>-0.000212</td>\n",
              "      <td>-0.000596</td>\n",
              "      <td>-0.000666</td>\n",
              "      <td>-0.000469</td>\n",
              "      <td>-0.000126</td>\n",
              "      <td>2.435148</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.999649</td>\n",
              "      <td>0.999944</td>\n",
              "      <td>0.999965</td>\n",
              "      <td>1.000196</td>\n",
              "      <td>0.999696</td>\n",
              "      <td>0.999958</td>\n",
              "      <td>0.999968</td>\n",
              "      <td>1.000220</td>\n",
              "      <td>0.999807</td>\n",
              "      <td>0.999904</td>\n",
              "      <td>0.999959</td>\n",
              "      <td>1.000203</td>\n",
              "      <td>0.999639</td>\n",
              "      <td>0.999957</td>\n",
              "      <td>1.000093</td>\n",
              "      <td>1.000232</td>\n",
              "      <td>0.999876</td>\n",
              "      <td>0.999964</td>\n",
              "      <td>1.000127</td>\n",
              "      <td>1.000264</td>\n",
              "      <td>1.000002</td>\n",
              "      <td>0.999981</td>\n",
              "      <td>1.000074</td>\n",
              "      <td>1.000218</td>\n",
              "      <td>1.000037</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000192</td>\n",
              "      <td>1.000271</td>\n",
              "      <td>0.999897</td>\n",
              "      <td>1.000022</td>\n",
              "      <td>1.000017</td>\n",
              "      <td>1.000271</td>\n",
              "      <td>0.999878</td>\n",
              "      <td>1.000013</td>\n",
              "      <td>1.000009</td>\n",
              "      <td>1.000288</td>\n",
              "      <td>1.876348</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>-2.234329</td>\n",
              "      <td>-2.473310</td>\n",
              "      <td>-2.780894</td>\n",
              "      <td>-2.624275</td>\n",
              "      <td>-2.223275</td>\n",
              "      <td>-2.457498</td>\n",
              "      <td>-2.947095</td>\n",
              "      <td>-2.824420</td>\n",
              "      <td>-2.146330</td>\n",
              "      <td>-2.444776</td>\n",
              "      <td>-2.936494</td>\n",
              "      <td>-2.812701</td>\n",
              "      <td>-2.226740</td>\n",
              "      <td>-2.471642</td>\n",
              "      <td>-2.958463</td>\n",
              "      <td>-2.833559</td>\n",
              "      <td>-2.145554</td>\n",
              "      <td>-2.452344</td>\n",
              "      <td>-2.939991</td>\n",
              "      <td>-2.815957</td>\n",
              "      <td>-2.217016</td>\n",
              "      <td>-2.441006</td>\n",
              "      <td>-2.925927</td>\n",
              "      <td>-2.803693</td>\n",
              "      <td>-2.221005</td>\n",
              "      <td>-2.472222</td>\n",
              "      <td>-2.962511</td>\n",
              "      <td>-2.825539</td>\n",
              "      <td>-2.219281</td>\n",
              "      <td>-2.457440</td>\n",
              "      <td>-2.940074</td>\n",
              "      <td>-2.815531</td>\n",
              "      <td>-2.218211</td>\n",
              "      <td>-2.448131</td>\n",
              "      <td>-2.930494</td>\n",
              "      <td>-2.808050</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>-0.690878</td>\n",
              "      <td>-0.550421</td>\n",
              "      <td>-0.858503</td>\n",
              "      <td>-0.719279</td>\n",
              "      <td>-0.674739</td>\n",
              "      <td>-0.534967</td>\n",
              "      <td>-0.846770</td>\n",
              "      <td>-0.712591</td>\n",
              "      <td>-0.661615</td>\n",
              "      <td>-0.520206</td>\n",
              "      <td>-0.832700</td>\n",
              "      <td>-0.758027</td>\n",
              "      <td>-0.682928</td>\n",
              "      <td>-0.546032</td>\n",
              "      <td>-0.858614</td>\n",
              "      <td>-0.720733</td>\n",
              "      <td>-0.668189</td>\n",
              "      <td>-0.531371</td>\n",
              "      <td>-0.846398</td>\n",
              "      <td>-0.714624</td>\n",
              "      <td>-0.656748</td>\n",
              "      <td>-0.518297</td>\n",
              "      <td>-0.833829</td>\n",
              "      <td>-0.758677</td>\n",
              "      <td>-0.674672</td>\n",
              "      <td>-0.542629</td>\n",
              "      <td>-0.855666</td>\n",
              "      <td>-0.719309</td>\n",
              "      <td>-0.662882</td>\n",
              "      <td>-0.531602</td>\n",
              "      <td>-0.844805</td>\n",
              "      <td>-0.715095</td>\n",
              "      <td>-0.651234</td>\n",
              "      <td>-0.519740</td>\n",
              "      <td>-0.834119</td>\n",
              "      <td>-0.761265</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>-0.102897</td>\n",
              "      <td>0.148811</td>\n",
              "      <td>0.102692</td>\n",
              "      <td>-0.084280</td>\n",
              "      <td>-0.084821</td>\n",
              "      <td>0.076748</td>\n",
              "      <td>0.113379</td>\n",
              "      <td>-0.079042</td>\n",
              "      <td>-0.141965</td>\n",
              "      <td>0.092157</td>\n",
              "      <td>0.068926</td>\n",
              "      <td>-0.073135</td>\n",
              "      <td>-0.094810</td>\n",
              "      <td>0.066663</td>\n",
              "      <td>0.101317</td>\n",
              "      <td>-0.086885</td>\n",
              "      <td>-0.077243</td>\n",
              "      <td>0.079847</td>\n",
              "      <td>0.050856</td>\n",
              "      <td>-0.084225</td>\n",
              "      <td>-0.136658</td>\n",
              "      <td>0.049776</td>\n",
              "      <td>0.062784</td>\n",
              "      <td>-0.077004</td>\n",
              "      <td>-0.085593</td>\n",
              "      <td>0.071332</td>\n",
              "      <td>0.047268</td>\n",
              "      <td>-0.087440</td>\n",
              "      <td>-0.069968</td>\n",
              "      <td>0.081165</td>\n",
              "      <td>0.053167</td>\n",
              "      <td>-0.084964</td>\n",
              "      <td>-0.128909</td>\n",
              "      <td>0.050011</td>\n",
              "      <td>0.064327</td>\n",
              "      <td>-0.079004</td>\n",
              "      <td>2.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>0.779075</td>\n",
              "      <td>0.848043</td>\n",
              "      <td>0.823588</td>\n",
              "      <td>0.497802</td>\n",
              "      <td>0.800057</td>\n",
              "      <td>0.863238</td>\n",
              "      <td>0.833490</td>\n",
              "      <td>0.501711</td>\n",
              "      <td>0.748864</td>\n",
              "      <td>0.835741</td>\n",
              "      <td>0.850335</td>\n",
              "      <td>0.506388</td>\n",
              "      <td>0.787369</td>\n",
              "      <td>0.854412</td>\n",
              "      <td>0.821265</td>\n",
              "      <td>0.494142</td>\n",
              "      <td>0.735308</td>\n",
              "      <td>0.865700</td>\n",
              "      <td>0.828476</td>\n",
              "      <td>0.493642</td>\n",
              "      <td>0.754923</td>\n",
              "      <td>0.880036</td>\n",
              "      <td>0.839849</td>\n",
              "      <td>0.499795</td>\n",
              "      <td>0.724391</td>\n",
              "      <td>0.860711</td>\n",
              "      <td>0.829810</td>\n",
              "      <td>0.491773</td>\n",
              "      <td>0.745289</td>\n",
              "      <td>0.869008</td>\n",
              "      <td>0.831409</td>\n",
              "      <td>0.492656</td>\n",
              "      <td>0.766507</td>\n",
              "      <td>0.838898</td>\n",
              "      <td>0.842981</td>\n",
              "      <td>0.498294</td>\n",
              "      <td>4.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>2.543020</td>\n",
              "      <td>2.333912</td>\n",
              "      <td>2.445605</td>\n",
              "      <td>3.778629</td>\n",
              "      <td>2.569812</td>\n",
              "      <td>2.348830</td>\n",
              "      <td>2.753787</td>\n",
              "      <td>3.933434</td>\n",
              "      <td>2.604758</td>\n",
              "      <td>2.060467</td>\n",
              "      <td>2.773804</td>\n",
              "      <td>3.930845</td>\n",
              "      <td>2.551725</td>\n",
              "      <td>2.342384</td>\n",
              "      <td>2.741127</td>\n",
              "      <td>3.769022</td>\n",
              "      <td>2.582013</td>\n",
              "      <td>2.044478</td>\n",
              "      <td>2.742619</td>\n",
              "      <td>3.908307</td>\n",
              "      <td>2.612386</td>\n",
              "      <td>2.059880</td>\n",
              "      <td>2.752623</td>\n",
              "      <td>3.908156</td>\n",
              "      <td>2.565264</td>\n",
              "      <td>2.088634</td>\n",
              "      <td>2.455091</td>\n",
              "      <td>3.756429</td>\n",
              "      <td>2.598145</td>\n",
              "      <td>2.050772</td>\n",
              "      <td>2.747083</td>\n",
              "      <td>3.905865</td>\n",
              "      <td>2.631955</td>\n",
              "      <td>2.066056</td>\n",
              "      <td>2.759666</td>\n",
              "      <td>3.909603</td>\n",
              "      <td>5.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8994e2aa-366d-4d81-b749-c8269e5a87c3')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-8994e2aa-366d-4d81-b749-c8269e5a87c3 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-8994e2aa-366d-4d81-b749-c8269e5a87c3');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-2aa4a4ff-1e7a-4dc7-abec-75e59b896fc8\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-2aa4a4ff-1e7a-4dc7-abec-75e59b896fc8')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-2aa4a4ff-1e7a-4dc7-abec-75e59b896fc8 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# print number of classes\n",
        "n_classes = df['class'].nunique()\n",
        "print('Number of classes: {}'.format(n_classes))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z8JjCRY7uOcI",
        "outputId": "0cc861d2-76ec-4072-cfc0-8a89cf53e8cb"
      },
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of classes: 6\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# print number of classes\n",
        "classes = np.sort(df['class'].unique())"
      ],
      "metadata": {
        "id": "kUDlDoMvuRLJ"
      },
      "execution_count": 45,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "classes"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cMFeSE6GuUZm",
        "outputId": "2731fa2e-aa02-4793-f327-780cae645bd9"
      },
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 1, 2, 3, 4, 5])"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Experiments"
      ],
      "metadata": {
        "id": "YoF4hQwqub80"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df.info()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XtSiKWmGuWP0",
        "outputId": "31870119-f542-4a4a-e383-3faa9ab2563e"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 6430 entries, 0 to 6429\n",
            "Data columns (total 37 columns):\n",
            " #   Column   Non-Null Count  Dtype  \n",
            "---  ------   --------------  -----  \n",
            " 0   Aattr    6430 non-null   float32\n",
            " 1   Battr    6430 non-null   float32\n",
            " 2   Cattr    6430 non-null   float32\n",
            " 3   Dattr    6430 non-null   float32\n",
            " 4   Eattr    6430 non-null   float32\n",
            " 5   Fattr    6430 non-null   float32\n",
            " 6   A1attr   6430 non-null   float32\n",
            " 7   B2attr   6430 non-null   float32\n",
            " 8   C3attr   6430 non-null   float32\n",
            " 9   D4attr   6430 non-null   float32\n",
            " 10  E5attr   6430 non-null   float32\n",
            " 11  F6attr   6430 non-null   float32\n",
            " 12  A7attr   6430 non-null   float32\n",
            " 13  B8attr   6430 non-null   float32\n",
            " 14  C9attr   6430 non-null   float32\n",
            " 15  D10attr  6430 non-null   float32\n",
            " 16  E11attr  6430 non-null   float32\n",
            " 17  F12attr  6430 non-null   float32\n",
            " 18  A13attr  6430 non-null   float32\n",
            " 19  B14attr  6430 non-null   float32\n",
            " 20  C15attr  6430 non-null   float32\n",
            " 21  D16attr  6430 non-null   float32\n",
            " 22  E17attr  6430 non-null   float32\n",
            " 23  F18attr  6430 non-null   float32\n",
            " 24  A19attr  6430 non-null   float32\n",
            " 25  B20attr  6430 non-null   float32\n",
            " 26  C21attr  6430 non-null   float32\n",
            " 27  D22attr  6430 non-null   float32\n",
            " 28  E23attr  6430 non-null   float32\n",
            " 29  F24attr  6430 non-null   float32\n",
            " 30  A25attr  6430 non-null   float32\n",
            " 31  B26attr  6430 non-null   float32\n",
            " 32  C27attr  6430 non-null   float32\n",
            " 33  D28attr  6430 non-null   float32\n",
            " 34  E29attr  6430 non-null   float32\n",
            " 35  F30attr  6430 non-null   float32\n",
            " 36  class    6430 non-null   int64  \n",
            "dtypes: float32(36), int64(1)\n",
            "memory usage: 954.6 KB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(f'Original df memory: {df.memory_usage(deep=True).sum()/1024/1024} MB')\n",
        "proposed_df = report_on_dataframe(df, unit=\"MB\")\n",
        "\n",
        "df = optimize_dtypes(df, proposed_df)\n",
        "print(f'Proposed df memory: {df.memory_usage(deep=True).sum()/1024/1024} MB')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RjMKaz-cvU7f",
        "outputId": "dae58380-a18c-4482-f2dc-8d60198ebd6a"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original df memory: 0.9322052001953125 MB\n",
            "Proposed df memory: 0.8892803192138672 MB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_train_cal, df_test = train_test_split(df, test_size = 500, random_state = 42, shuffle = True)\n",
        "df_proper_train, df_cal = train_test_split(df_train_cal, test_size = 500, random_state = 42, shuffle = True)"
      ],
      "metadata": {
        "id": "RjQB5Pnzw5PW"
      },
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Proper training set size: {}'.format(len(df_proper_train)))\n",
        "print('Calibration set size: {}'.format(len(df_cal)))\n",
        "print('Test set size: {}'.format(len(df_test)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-sAJW-KZvicf",
        "outputId": "9f5ca64b-7353-4634-c7d1-4c026427cbcd"
      },
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Proper training set size: 5430\n",
            "Calibration set size: 500\n",
            "Test set size: 500\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "X_train = df_train_cal.drop('class', axis=1)\n",
        "y_train = df_train_cal['class']\n",
        "\n",
        "X_proper_train = df_proper_train.drop('class', axis=1)\n",
        "y_proper_train = df_proper_train['class']\n",
        "\n",
        "X_cal = df_cal.drop('class', axis=1)\n",
        "y_cal = df_cal['class']\n",
        "\n",
        "X_test = df_test.drop('class', axis=1)\n",
        "y_test = df_test['class']"
      ],
      "metadata": {
        "id": "lofnh93HxYF3"
      },
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "lb = LabelBinarizer()\n",
        "y_test_binary = lb.fit_transform(y_test)"
      ],
      "metadata": {
        "id": "yYlgJTC2xl9m"
      },
      "execution_count": 52,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "y_test_binary"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wr9od_CSxqyR",
        "outputId": "7befcfdd-4533-43d6-b5f0-d7f743ee9af7"
      },
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0, 1, 0, 0, 0, 0],\n",
              "       [0, 0, 1, 0, 0, 0],\n",
              "       [0, 0, 0, 0, 0, 1],\n",
              "       ...,\n",
              "       [1, 0, 0, 0, 0, 0],\n",
              "       [0, 0, 1, 0, 0, 0],\n",
              "       [0, 0, 0, 0, 0, 1]])"
            ]
          },
          "metadata": {},
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def brier_loss_calc(y_true, prob):\n",
        "    return ((y_true - prob)**2).mean()"
      ],
      "metadata": {
        "id": "H5UBsrCD37zP"
      },
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "This code illustrates the multi-class Venn-ABERS calibration as implemented in \"Multi-class probabilistic classification using inductive and cross Venn–Abers predictors.\" (Manokhin, Valery) In Conformal and Probabilistic Prediction and Applications, pp. 228-240. PMLR, 2017."
      ],
      "metadata": {
        "id": "Tl5UOOXyx1G7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Creating a dataframe to record the performance of various models\n",
        "model_names = ['Naive Bayes', 'Logistic Regression', 'Random Forest Classifier', 'AdaBoost Classifier', 'CatBoost Classifier', 'SVC', 'LGBM Classifier', 'XGBoost Classifier']\n",
        "\n",
        "methods = ['uncalibrated',\t'Platt',\t'Isotonic',\t'Platt-CV',\t'Isotonic-CV',\t'IVAP',\t'CVAP']\n",
        "\n",
        "accuracy_df = pd.DataFrame(index=model_names, columns=methods)\n",
        "\n",
        "# setup dataframes to store results for the Brier and the Log loss\n",
        "brier_loss_df = accuracy_df.copy()\n",
        "log_loss_df = accuracy_df.copy()\n",
        "\n",
        "# Create a DataFrame to record execution times\n",
        "time_df = pd.DataFrame(index=model_names, columns=['Execution Time (s)'])"
      ],
      "metadata": {
        "id": "Wbw2MPbAy3mf"
      },
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "RANDOM_STATE = 42"
      ],
      "metadata": {
        "id": "o_Kovh4W0U5k"
      },
      "execution_count": 56,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate_model_performance(model, model_name):\n",
        "\n",
        "  start_time = time.time()  # Record start time\n",
        "  model.fit(X_train, y_train)\n",
        "\n",
        "  # Predict class score on the test set\n",
        "  p_pred = model.predict_proba(X_test)\n",
        "  # Predict on the test set\n",
        "  y_pred = model.predict(X_test)\n",
        "\n",
        "  # compute results of the underlying classifier\n",
        "  accuracy_df.loc[model_name, 'uncalibrated'] = accuracy_score(y_test, y_pred)\n",
        "  log_loss_df.loc[model_name, 'uncalibrated'] = log_loss(y_test, p_pred)\n",
        "  brier_loss_df.loc[model_name, 'uncalibrated'] = brier_loss_calc(y_test_binary, p_pred)\n",
        "\n",
        "  # compute results for Platt calibrator\n",
        "  model.fit(X_proper_train, y_proper_train)\n",
        "  cal_sigm = CalibratedClassifierCV(model, method='sigmoid', cv='prefit')\n",
        "  cal_sigm.fit(X_cal, y_cal)\n",
        "\n",
        "  #compute class and probability of predicted class\n",
        "  p_pred = cal_sigm.predict_proba(X_test)\n",
        "  y_pred = cal_sigm.predict(X_test)\n",
        "\n",
        "  accuracy_df.loc[model_name, 'Platt'] = accuracy_score(y_test, y_pred)\n",
        "  log_loss_df.loc[model_name, 'Platt'] = log_loss(y_test, p_pred)\n",
        "  brier_loss_df.loc[model_name, 'Platt'] = brier_loss_calc(y_test_binary, p_pred)\n",
        "\n",
        "  # compute results for Isotonic regression calibrator\n",
        "  cal_iso = CalibratedClassifierCV(model, method='isotonic', cv='prefit')\n",
        "  cal_iso.fit(X_cal, y_cal)\n",
        "  p_pred = cal_iso.predict_proba(X_test)\n",
        "  y_pred = cal_iso.predict(X_test)\n",
        "\n",
        "  accuracy_df.loc[model_name, 'Isotonic'] = accuracy_score(y_test, y_pred)\n",
        "  log_loss_df.loc[model_name, 'Isotonic'] = log_loss(y_test, p_pred)\n",
        "  brier_loss_df.loc[model_name, 'Isotonic'] = brier_loss_calc(y_test_binary, p_pred)\n",
        "\n",
        "  # compute results for Platt-CV calibrator\n",
        "  cal_sigm_cv = CalibratedClassifierCV(model, method='sigmoid', cv=5)\n",
        "  cal_sigm_cv.fit(X_train, y_train)\n",
        "\n",
        "  #compute class and probability of predicted class\n",
        "  p_pred = cal_sigm_cv.predict_proba(X_test)\n",
        "  y_pred = cal_sigm_cv.predict(X_test)\n",
        "\n",
        "  accuracy_df.loc[model_name, 'Platt-CV'] = accuracy_score(y_test, y_pred)\n",
        "  log_loss_df.loc[model_name, 'Platt-CV'] = log_loss(y_test, p_pred)\n",
        "  brier_loss_df.loc[model_name, 'Platt-CV'] = brier_loss_calc(y_test_binary, p_pred)\n",
        "\n",
        "  # compute results for Isotonic-CV calibrator\n",
        "  cal_iso_cv = CalibratedClassifierCV(model, method='isotonic', cv=5)\n",
        "  cal_iso_cv.fit(X_train, y_train)\n",
        "  p_pred = cal_iso_cv.predict_proba(X_test)\n",
        "  y_pred = cal_iso_cv.predict(X_test)\n",
        "\n",
        "  accuracy_df.loc[model_name, 'Isotonic-CV'] = accuracy_score(y_test, y_pred)\n",
        "  log_loss_df.loc[model_name, 'Isotonic-CV'] = log_loss(y_test, p_pred)\n",
        "  brier_loss_df.loc[model_name, 'Isotonic-CV'] = brier_loss_calc(y_test_binary, p_pred)\n",
        "\n",
        "  # compute results for IVAP calibrator\n",
        "  va = VennAbersCalibrator(model, inductive=True, cal_size=500, random_state=RANDOM_STATE)\n",
        "  va.fit(np.asarray(X_train), np.asarray(y_train))\n",
        "\n",
        "  p_pred_va = va.predict_proba(np.array(X_test))\n",
        "  y_pred = np.argmax(va.predict(X_test), axis=1)\n",
        "\n",
        "  accuracy_df.loc[model_name, 'IVAP'] = accuracy_score(y_test, y_pred)\n",
        "  log_loss_df.loc[model_name, 'IVAP'] = log_loss(y_test, p_pred_va)\n",
        "  brier_loss_df.loc[model_name, 'IVAP'] = brier_loss_calc(y_test_binary, p_pred_va)\n",
        "\n",
        "  # compute results for CVAP calibrator\n",
        "  va_cv = VennAbersCalibrator(model, inductive=False, n_splits=5)\n",
        "  va_cv.fit(np.asarray(X_train), np.asarray(y_train))\n",
        "  p_pred_cv = va_cv.predict_proba(np.asarray(X_test))\n",
        "  y_pred =  np.argmax(va_cv.predict(X_test), axis=1)\n",
        "\n",
        "  accuracy_df.loc[model_name, 'CVAP'] = accuracy_score(y_test, y_pred)\n",
        "  log_loss_df.loc[model_name, 'CVAP'] = log_loss(y_test, p_pred_cv)\n",
        "  brier_loss_df.loc[model_name, 'CVAP'] = brier_loss_calc(y_test_binary, p_pred_cv)\n",
        "\n",
        "  end_time = time.time()  # Record end time\n",
        "  execution_time = end_time - start_time  # Calculate execution time in seconds\n",
        "\n",
        "  # Record execution time in the time DataFrame\n",
        "  time_df.loc[model_name, 'Execution Time (s)'] = execution_time\n"
      ],
      "metadata": {
        "id": "16nkDV8q0zPM"
      },
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "models = [\n",
        "    GaussianNB(),\n",
        "    LogisticRegression(n_jobs=-1),\n",
        "    RandomForestClassifier(n_jobs=-1),\n",
        "    AdaBoostClassifier(),\n",
        "    CatBoostClassifier(task_type=\"GPU\" if is_cuda_available else \"CPU\",\n",
        "                       thread_count=-1, verbose=0, random_state=RANDOM_STATE),\n",
        "    SVC(probability=True),\n",
        "    LGBMClassifier(device='gpu' if is_cuda_available else 'cpu',\n",
        "                   random_state=RANDOM_STATE, n_jobs=-1, verbose=-1),\n",
        "    XGBClassifier(tree_method='gpu_hist' if is_cuda_available else 'auto',\n",
        "                  n_jobs=-1, num_class=n_classes)\n",
        "]"
      ],
      "metadata": {
        "id": "6jEheEF5FEqh"
      },
      "execution_count": 58,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for model, model_name in zip(models, model_names):\n",
        "    print(f'processing: {model_name}')\n",
        "    evaluate_model_performance(model, model_name)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "llFEtErQIDXk",
        "outputId": "da2ebbcb-144e-46e3-c876-46ca8ac94ccd"
      },
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "processing: Naive Bayes\n",
            "processing: Logistic Regression\n",
            "processing: Random Forest Classifier\n",
            "processing: AdaBoost Classifier\n",
            "processing: CatBoost Classifier\n",
            "processing: SVC\n",
            "processing: LGBM Classifier\n",
            "processing: XGBoost Classifier\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 297
        },
        "id": "xntZlnRA7H_9",
        "outputId": "705816d6-4ae3-45a1-f5e9-7fabb0eb3bd0"
      },
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                         uncalibrated  Platt Isotonic Platt-CV Isotonic-CV  \\\n",
              "Naive Bayes                     0.804  0.806    0.826    0.806       0.822   \n",
              "Logistic Regression             0.858  0.802    0.836      0.8        0.81   \n",
              "Random Forest Classifier         0.91  0.906    0.904    0.914       0.904   \n",
              "AdaBoost Classifier             0.758  0.628    0.746    0.724        0.78   \n",
              "CatBoost Classifier              0.92  0.908    0.894    0.916       0.918   \n",
              "SVC                             0.896  0.894    0.886    0.894        0.89   \n",
              "LGBM Classifier                  0.93  0.922    0.918    0.924        0.93   \n",
              "XGBoost Classifier              0.916  0.918    0.906    0.924       0.924   \n",
              "\n",
              "                           IVAP   CVAP  \n",
              "Naive Bayes               0.834   0.83  \n",
              "Logistic Regression       0.878  0.882  \n",
              "Random Forest Classifier    0.9  0.904  \n",
              "AdaBoost Classifier       0.892  0.876  \n",
              "CatBoost Classifier       0.906  0.926  \n",
              "SVC                         0.9  0.896  \n",
              "LGBM Classifier           0.884  0.918  \n",
              "XGBoost Classifier        0.894   0.91  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-f589cdcf-1a2c-4ad8-9a00-8167e6befaa1\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>uncalibrated</th>\n",
              "      <th>Platt</th>\n",
              "      <th>Isotonic</th>\n",
              "      <th>Platt-CV</th>\n",
              "      <th>Isotonic-CV</th>\n",
              "      <th>IVAP</th>\n",
              "      <th>CVAP</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Naive Bayes</th>\n",
              "      <td>0.804</td>\n",
              "      <td>0.806</td>\n",
              "      <td>0.826</td>\n",
              "      <td>0.806</td>\n",
              "      <td>0.822</td>\n",
              "      <td>0.834</td>\n",
              "      <td>0.83</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Logistic Regression</th>\n",
              "      <td>0.858</td>\n",
              "      <td>0.802</td>\n",
              "      <td>0.836</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.81</td>\n",
              "      <td>0.878</td>\n",
              "      <td>0.882</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Random Forest Classifier</th>\n",
              "      <td>0.91</td>\n",
              "      <td>0.906</td>\n",
              "      <td>0.904</td>\n",
              "      <td>0.914</td>\n",
              "      <td>0.904</td>\n",
              "      <td>0.9</td>\n",
              "      <td>0.904</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>AdaBoost Classifier</th>\n",
              "      <td>0.758</td>\n",
              "      <td>0.628</td>\n",
              "      <td>0.746</td>\n",
              "      <td>0.724</td>\n",
              "      <td>0.78</td>\n",
              "      <td>0.892</td>\n",
              "      <td>0.876</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>CatBoost Classifier</th>\n",
              "      <td>0.92</td>\n",
              "      <td>0.908</td>\n",
              "      <td>0.894</td>\n",
              "      <td>0.916</td>\n",
              "      <td>0.918</td>\n",
              "      <td>0.906</td>\n",
              "      <td>0.926</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>SVC</th>\n",
              "      <td>0.896</td>\n",
              "      <td>0.894</td>\n",
              "      <td>0.886</td>\n",
              "      <td>0.894</td>\n",
              "      <td>0.89</td>\n",
              "      <td>0.9</td>\n",
              "      <td>0.896</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>LGBM Classifier</th>\n",
              "      <td>0.93</td>\n",
              "      <td>0.922</td>\n",
              "      <td>0.918</td>\n",
              "      <td>0.924</td>\n",
              "      <td>0.93</td>\n",
              "      <td>0.884</td>\n",
              "      <td>0.918</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>XGBoost Classifier</th>\n",
              "      <td>0.916</td>\n",
              "      <td>0.918</td>\n",
              "      <td>0.906</td>\n",
              "      <td>0.924</td>\n",
              "      <td>0.924</td>\n",
              "      <td>0.894</td>\n",
              "      <td>0.91</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f589cdcf-1a2c-4ad8-9a00-8167e6befaa1')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-f589cdcf-1a2c-4ad8-9a00-8167e6befaa1 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-f589cdcf-1a2c-4ad8-9a00-8167e6befaa1');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-d1e418a5-c647-4c31-abe3-950818fc2359\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-d1e418a5-c647-4c31-abe3-950818fc2359')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-d1e418a5-c647-4c31-abe3-950818fc2359 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "log_loss_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 297
        },
        "id": "UsxPyCp1GF1a",
        "outputId": "48c62db3-69de-4050-a9bc-ac8390bf4b2f"
      },
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                         uncalibrated     Platt  Isotonic  Platt-CV  \\\n",
              "Naive Bayes                  3.613003  0.748671  0.771357  0.733909   \n",
              "Logistic Regression          0.330626  0.573444  0.676732   0.56213   \n",
              "Random Forest Classifier     0.262673  0.285431   0.52146  0.244604   \n",
              "AdaBoost Classifier          1.149778  1.016316  1.610103  0.947233   \n",
              "CatBoost Classifier          0.212955  0.314463  0.562956  0.263331   \n",
              "SVC                          0.267848  0.352296  0.582236  0.315861   \n",
              "LGBM Classifier               0.23341  0.317691  0.429046  0.276056   \n",
              "XGBoost Classifier           0.257202    0.3323  0.440867  0.275185   \n",
              "\n",
              "                         Isotonic-CV      IVAP      CVAP  \n",
              "Naive Bayes                 0.544072  0.455508  0.472773  \n",
              "Logistic Regression         0.486974  0.326944  0.353028  \n",
              "Random Forest Classifier    0.224666  0.274455  0.280479  \n",
              "AdaBoost Classifier           0.7691  0.301958  0.332863  \n",
              "CatBoost Classifier         0.208423  0.267735  0.272995  \n",
              "SVC                         0.289229  0.261901  0.305008  \n",
              "LGBM Classifier             0.198108  0.268679  0.275074  \n",
              "XGBoost Classifier             0.209  0.267851  0.278991  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b42477e3-9cca-49f3-8f83-e57231965778\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>uncalibrated</th>\n",
              "      <th>Platt</th>\n",
              "      <th>Isotonic</th>\n",
              "      <th>Platt-CV</th>\n",
              "      <th>Isotonic-CV</th>\n",
              "      <th>IVAP</th>\n",
              "      <th>CVAP</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Naive Bayes</th>\n",
              "      <td>3.613003</td>\n",
              "      <td>0.748671</td>\n",
              "      <td>0.771357</td>\n",
              "      <td>0.733909</td>\n",
              "      <td>0.544072</td>\n",
              "      <td>0.455508</td>\n",
              "      <td>0.472773</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Logistic Regression</th>\n",
              "      <td>0.330626</td>\n",
              "      <td>0.573444</td>\n",
              "      <td>0.676732</td>\n",
              "      <td>0.56213</td>\n",
              "      <td>0.486974</td>\n",
              "      <td>0.326944</td>\n",
              "      <td>0.353028</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Random Forest Classifier</th>\n",
              "      <td>0.262673</td>\n",
              "      <td>0.285431</td>\n",
              "      <td>0.52146</td>\n",
              "      <td>0.244604</td>\n",
              "      <td>0.224666</td>\n",
              "      <td>0.274455</td>\n",
              "      <td>0.280479</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>AdaBoost Classifier</th>\n",
              "      <td>1.149778</td>\n",
              "      <td>1.016316</td>\n",
              "      <td>1.610103</td>\n",
              "      <td>0.947233</td>\n",
              "      <td>0.7691</td>\n",
              "      <td>0.301958</td>\n",
              "      <td>0.332863</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>CatBoost Classifier</th>\n",
              "      <td>0.212955</td>\n",
              "      <td>0.314463</td>\n",
              "      <td>0.562956</td>\n",
              "      <td>0.263331</td>\n",
              "      <td>0.208423</td>\n",
              "      <td>0.267735</td>\n",
              "      <td>0.272995</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>SVC</th>\n",
              "      <td>0.267848</td>\n",
              "      <td>0.352296</td>\n",
              "      <td>0.582236</td>\n",
              "      <td>0.315861</td>\n",
              "      <td>0.289229</td>\n",
              "      <td>0.261901</td>\n",
              "      <td>0.305008</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>LGBM Classifier</th>\n",
              "      <td>0.23341</td>\n",
              "      <td>0.317691</td>\n",
              "      <td>0.429046</td>\n",
              "      <td>0.276056</td>\n",
              "      <td>0.198108</td>\n",
              "      <td>0.268679</td>\n",
              "      <td>0.275074</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>XGBoost Classifier</th>\n",
              "      <td>0.257202</td>\n",
              "      <td>0.3323</td>\n",
              "      <td>0.440867</td>\n",
              "      <td>0.275185</td>\n",
              "      <td>0.209</td>\n",
              "      <td>0.267851</td>\n",
              "      <td>0.278991</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b42477e3-9cca-49f3-8f83-e57231965778')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-b42477e3-9cca-49f3-8f83-e57231965778 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-b42477e3-9cca-49f3-8f83-e57231965778');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-c27760c6-211e-43bf-9733-c17e46df1e1f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c27760c6-211e-43bf-9733-c17e46df1e1f')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-c27760c6-211e-43bf-9733-c17e46df1e1f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "brier_loss_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 297
        },
        "id": "i1ZIt0xoBkEr",
        "outputId": "3251a4f7-cc30-49fa-ec23-b990e63be8d0"
      },
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                         uncalibrated     Platt  Isotonic  Platt-CV  \\\n",
              "Naive Bayes                  0.062693  0.053826  0.046361   0.05356   \n",
              "Logistic Regression          0.031796  0.048106  0.043621   0.04805   \n",
              "Random Forest Classifier     0.023633  0.022965  0.024033  0.021033   \n",
              "AdaBoost Classifier          0.095207  0.084701  0.073049  0.078731   \n",
              "CatBoost Classifier           0.02027  0.023712  0.023066  0.021605   \n",
              "SVC                          0.025388  0.031599  0.029787  0.028589   \n",
              "LGBM Classifier              0.019972  0.022485  0.021968  0.020342   \n",
              "XGBoost Classifier           0.022222  0.024277  0.023507  0.020842   \n",
              "\n",
              "                         Isotonic-CV      IVAP      CVAP  \n",
              "Naive Bayes                 0.043806  0.040992  0.041521  \n",
              "Logistic Regression         0.044531  0.030174  0.030698  \n",
              "Random Forest Classifier    0.021669  0.023952  0.023954  \n",
              "AdaBoost Classifier         0.065862  0.027373  0.028507  \n",
              "CatBoost Classifier         0.020099  0.022692  0.022404  \n",
              "SVC                         0.027174  0.023962  0.025891  \n",
              "LGBM Classifier             0.018867  0.024562  0.022684  \n",
              "XGBoost Classifier          0.019788  0.023448  0.023331  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2156203e-facc-4448-88eb-7985873fc217\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>uncalibrated</th>\n",
              "      <th>Platt</th>\n",
              "      <th>Isotonic</th>\n",
              "      <th>Platt-CV</th>\n",
              "      <th>Isotonic-CV</th>\n",
              "      <th>IVAP</th>\n",
              "      <th>CVAP</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Naive Bayes</th>\n",
              "      <td>0.062693</td>\n",
              "      <td>0.053826</td>\n",
              "      <td>0.046361</td>\n",
              "      <td>0.05356</td>\n",
              "      <td>0.043806</td>\n",
              "      <td>0.040992</td>\n",
              "      <td>0.041521</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Logistic Regression</th>\n",
              "      <td>0.031796</td>\n",
              "      <td>0.048106</td>\n",
              "      <td>0.043621</td>\n",
              "      <td>0.04805</td>\n",
              "      <td>0.044531</td>\n",
              "      <td>0.030174</td>\n",
              "      <td>0.030698</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Random Forest Classifier</th>\n",
              "      <td>0.023633</td>\n",
              "      <td>0.022965</td>\n",
              "      <td>0.024033</td>\n",
              "      <td>0.021033</td>\n",
              "      <td>0.021669</td>\n",
              "      <td>0.023952</td>\n",
              "      <td>0.023954</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>AdaBoost Classifier</th>\n",
              "      <td>0.095207</td>\n",
              "      <td>0.084701</td>\n",
              "      <td>0.073049</td>\n",
              "      <td>0.078731</td>\n",
              "      <td>0.065862</td>\n",
              "      <td>0.027373</td>\n",
              "      <td>0.028507</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>CatBoost Classifier</th>\n",
              "      <td>0.02027</td>\n",
              "      <td>0.023712</td>\n",
              "      <td>0.023066</td>\n",
              "      <td>0.021605</td>\n",
              "      <td>0.020099</td>\n",
              "      <td>0.022692</td>\n",
              "      <td>0.022404</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>SVC</th>\n",
              "      <td>0.025388</td>\n",
              "      <td>0.031599</td>\n",
              "      <td>0.029787</td>\n",
              "      <td>0.028589</td>\n",
              "      <td>0.027174</td>\n",
              "      <td>0.023962</td>\n",
              "      <td>0.025891</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>LGBM Classifier</th>\n",
              "      <td>0.019972</td>\n",
              "      <td>0.022485</td>\n",
              "      <td>0.021968</td>\n",
              "      <td>0.020342</td>\n",
              "      <td>0.018867</td>\n",
              "      <td>0.024562</td>\n",
              "      <td>0.022684</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>XGBoost Classifier</th>\n",
              "      <td>0.022222</td>\n",
              "      <td>0.024277</td>\n",
              "      <td>0.023507</td>\n",
              "      <td>0.020842</td>\n",
              "      <td>0.019788</td>\n",
              "      <td>0.023448</td>\n",
              "      <td>0.023331</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2156203e-facc-4448-88eb-7985873fc217')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-2156203e-facc-4448-88eb-7985873fc217 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-2156203e-facc-4448-88eb-7985873fc217');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-908d4b75-0f02-4a13-98e4-96e279c366ab\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-908d4b75-0f02-4a13-98e4-96e279c366ab')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-908d4b75-0f02-4a13-98e4-96e279c366ab button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 62
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "time_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 297
        },
        "id": "wMRQM-BGLC56",
        "outputId": "35a5150d-3d58-46b5-bc25-56018f7e0d23"
      },
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                         Execution Time (s)\n",
              "Naive Bayes                        3.891745\n",
              "Logistic Regression               35.032923\n",
              "Random Forest Classifier           57.76286\n",
              "AdaBoost Classifier               40.594322\n",
              "CatBoost Classifier              623.017915\n",
              "SVC                               40.488699\n",
              "LGBM Classifier                   38.485523\n",
              "XGBoost Classifier               141.783125"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-446fd235-4930-43c9-893c-a53d91b7fe0b\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Execution Time (s)</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Naive Bayes</th>\n",
              "      <td>3.891745</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Logistic Regression</th>\n",
              "      <td>35.032923</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Random Forest Classifier</th>\n",
              "      <td>57.76286</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>AdaBoost Classifier</th>\n",
              "      <td>40.594322</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>CatBoost Classifier</th>\n",
              "      <td>623.017915</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>SVC</th>\n",
              "      <td>40.488699</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>LGBM Classifier</th>\n",
              "      <td>38.485523</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>XGBoost Classifier</th>\n",
              "      <td>141.783125</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-446fd235-4930-43c9-893c-a53d91b7fe0b')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-446fd235-4930-43c9-893c-a53d91b7fe0b button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-446fd235-4930-43c9-893c-a53d91b7fe0b');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-0a99b2d2-c1c1-4eee-90ba-c1f9805159e9\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-0a99b2d2-c1c1-4eee-90ba-c1f9805159e9')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-0a99b2d2-c1c1-4eee-90ba-c1f9805159e9 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 63
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_df.to_csv('accuracy.csv')\n",
        "log_loss_df.to_csv('log_loss.csv')\n",
        "brier_loss_df.to_csv('brier_loss.csv')\n",
        "time_df.to_csv('time_cpu.csv')"
      ],
      "metadata": {
        "id": "CkOrkJB1VkYs"
      },
      "execution_count": 64,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Mean Brier and log loss scores"
      ],
      "metadata": {
        "id": "FW90HjPGgGMX"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Conformal Prediction methods, IVAP and CVAP calibrators achieve the best calibration as indicated by lower Brier score and log loss."
      ],
      "metadata": {
        "id": "_1Z9yt7AkDuf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "log_loss_df.mean().sort_values()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4upBcXFAgG-R",
        "outputId": "70c04a45-f253-411a-8479-81566795e231"
      },
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "IVAP            0.303129\n",
              "CVAP            0.321401\n",
              "Isotonic-CV     0.366197\n",
              "Platt-CV        0.452289\n",
              "Platt           0.492576\n",
              "Isotonic        0.699345\n",
              "uncalibrated    0.790937\n",
              "dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "brier_loss_df.mean().sort_values()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2V8K2V9zgMr1",
        "outputId": "6ff6a5e2-58e4-47e9-db40-e3a28b959fea"
      },
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "IVAP            0.027144\n",
              "CVAP            0.027374\n",
              "Isotonic-CV     0.032724\n",
              "Isotonic        0.035674\n",
              "Platt-CV        0.036594\n",
              "uncalibrated    0.037648\n",
              "Platt           0.038959\n",
              "dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_df.mean().sort_values(ascending=False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pgiktRanj956",
        "outputId": "ba8894b5-0e99-4355-f0be-551cdf4dbce4"
      },
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "CVAP            0.89275\n",
              "IVAP            0.88600\n",
              "uncalibrated    0.87400\n",
              "Isotonic-CV     0.87225\n",
              "Isotonic        0.86450\n",
              "Platt-CV        0.86275\n",
              "Platt           0.84800\n",
              "dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 70
        }
      ]
    }
  ]
}