Chapter_06.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Inductive Conformal Prediction for binary classification"
      ],
      "metadata": {
        "id": "5g8JAwEGo-p-"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "Conformal Prediction is a machine learning framework that allows to quantify prediction uncertainty to produce reliable probabilisic predictions. Conformal Prediction can be used on top of any underlying model and works with any data distribution and any dataset size.\n",
        "\n",
        "The goal of this notebook is to help you understand Inductive Conformal Prediction for classification problem."
      ],
      "metadata": {
        "id": "Ohs_6mZOtAdr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#install OpenML\n",
        "!pip install nonconformist\n",
        "!pip install openml\n",
        "!pip install dtype_diet\n",
        "\n",
        "!pip install ipywidgets\n",
        "!jupyter nbextension enable --py widgetsnbextension"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ig67InlUpbFR",
        "outputId": "235afbee-6c6e-490b-e05d-7e1abacd775c"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: nonconformist in /usr/local/lib/python3.10/dist-packages (2.1.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from nonconformist) (1.22.4)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from nonconformist) (1.2.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from nonconformist) (1.10.1)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from nonconformist) (1.5.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->nonconformist) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->nonconformist) (2022.7.1)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->nonconformist) (1.2.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->nonconformist) (3.1.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->nonconformist) (1.16.0)\n",
            "Requirement already satisfied: openml in /usr/local/lib/python3.10/dist-packages (0.14.0)\n",
            "Requirement already satisfied: liac-arff>=2.4.0 in /usr/local/lib/python3.10/dist-packages (from openml) (2.5.0)\n",
            "Requirement already satisfied: xmltodict in /usr/local/lib/python3.10/dist-packages (from openml) (0.13.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from openml) (2.27.1)\n",
            "Requirement already satisfied: scikit-learn>=0.18 in /usr/local/lib/python3.10/dist-packages (from openml) (1.2.2)\n",
            "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.10/dist-packages (from openml) (2.8.2)\n",
            "Requirement already satisfied: pandas>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from openml) (1.5.3)\n",
            "Requirement already satisfied: scipy>=0.13.3 in /usr/local/lib/python3.10/dist-packages (from openml) (1.10.1)\n",
            "Requirement already satisfied: numpy>=1.6.2 in /usr/local/lib/python3.10/dist-packages (from openml) (1.22.4)\n",
            "Requirement already satisfied: minio in /usr/local/lib/python3.10/dist-packages (from openml) (7.1.15)\n",
            "Requirement already satisfied: pyarrow in /usr/local/lib/python3.10/dist-packages (from openml) (9.0.0)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0.0->openml) (2022.7.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil->openml) (1.16.0)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.18->openml) (1.2.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.18->openml) (3.1.0)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from minio->openml) (2023.5.7)\n",
            "Requirement already satisfied: urllib3 in /usr/local/lib/python3.10/dist-packages (from minio->openml) (1.26.16)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->openml) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->openml) (3.4)\n",
            "Requirement already satisfied: dtype_diet in /usr/local/lib/python3.10/dist-packages (0.0.1)\n",
            "Requirement already satisfied: pandas>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from dtype_diet) (1.5.3)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.10/dist-packages (from dtype_diet) (0.8.10)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0.0->dtype_diet) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0.0->dtype_diet) (2022.7.1)\n",
            "Requirement already satisfied: numpy>=1.21.0 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0.0->dtype_diet) (1.22.4)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas>=1.0.0->dtype_diet) (1.16.0)\n",
            "Requirement already satisfied: ipywidgets in /usr/local/lib/python3.10/dist-packages (7.7.1)\n",
            "Requirement already satisfied: ipykernel>=4.5.1 in /usr/local/lib/python3.10/dist-packages (from ipywidgets) (5.5.6)\n",
            "Requirement already satisfied: ipython-genutils~=0.2.0 in /usr/local/lib/python3.10/dist-packages (from ipywidgets) (0.2.0)\n",
            "Requirement already satisfied: traitlets>=4.3.1 in /usr/local/lib/python3.10/dist-packages (from ipywidgets) (5.7.1)\n",
            "Requirement already satisfied: widgetsnbextension~=3.6.0 in /usr/local/lib/python3.10/dist-packages (from ipywidgets) (3.6.4)\n",
            "Requirement already satisfied: ipython>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from ipywidgets) (7.34.0)\n",
            "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from ipywidgets) (3.0.7)\n",
            "Requirement already satisfied: jupyter-client in /usr/local/lib/python3.10/dist-packages (from ipykernel>=4.5.1->ipywidgets) (6.1.12)\n",
            "Requirement already satisfied: tornado>=4.2 in /usr/local/lib/python3.10/dist-packages (from ipykernel>=4.5.1->ipywidgets) (6.3.1)\n",
            "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets) (67.7.2)\n",
            "Requirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets) (0.18.2)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets) (4.4.2)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets) (0.7.5)\n",
            "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets) (3.0.38)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets) (2.14.0)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets) (0.2.0)\n",
            "Requirement already satisfied: matplotlib-inline in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets) (0.1.6)\n",
            "Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.10/dist-packages (from ipython>=4.0.0->ipywidgets) (4.8.0)\n",
            "Requirement already satisfied: notebook>=4.4.1 in /usr/local/lib/python3.10/dist-packages (from widgetsnbextension~=3.6.0->ipywidgets) (6.4.8)\n",
            "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /usr/local/lib/python3.10/dist-packages (from jedi>=0.16->ipython>=4.0.0->ipywidgets) (0.8.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (3.1.2)\n",
            "Requirement already satisfied: pyzmq>=17 in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (23.2.1)\n",
            "Requirement already satisfied: argon2-cffi in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (21.3.0)\n",
            "Requirement already satisfied: jupyter-core>=4.6.1 in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (5.3.1)\n",
            "Requirement already satisfied: nbformat in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (5.9.0)\n",
            "Requirement already satisfied: nbconvert in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (6.5.4)\n",
            "Requirement already satisfied: nest-asyncio>=1.5 in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.5.6)\n",
            "Requirement already satisfied: Send2Trash>=1.8.0 in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.8.2)\n",
            "Requirement already satisfied: terminado>=0.8.3 in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.17.1)\n",
            "Requirement already satisfied: prometheus-client in /usr/local/lib/python3.10/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.17.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.10/dist-packages (from jupyter-client->ipykernel>=4.5.1->ipywidgets) (2.8.2)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.10/dist-packages (from pexpect>4.3->ipython>=4.0.0->ipywidgets) (0.7.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/dist-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython>=4.0.0->ipywidgets) (0.2.6)\n",
            "Requirement already satisfied: platformdirs>=2.5 in /usr/local/lib/python3.10/dist-packages (from jupyter-core>=4.6.1->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (3.7.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.1->jupyter-client->ipykernel>=4.5.1->ipywidgets) (1.16.0)\n",
            "Requirement already satisfied: argon2-cffi-bindings in /usr/local/lib/python3.10/dist-packages (from argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (21.2.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (2.1.3)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (4.9.2)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (4.11.2)\n",
            "Requirement already satisfied: bleach in /usr/local/lib/python3.10/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (6.0.0)\n",
            "Requirement already satisfied: defusedxml in /usr/local/lib/python3.10/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.7.1)\n",
            "Requirement already satisfied: entrypoints>=0.2.2 in /usr/local/lib/python3.10/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.4)\n",
            "Requirement already satisfied: jupyterlab-pygments in /usr/local/lib/python3.10/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.2.2)\n",
            "Requirement already satisfied: mistune<2,>=0.8.1 in /usr/local/lib/python3.10/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.8.4)\n",
            "Requirement already satisfied: nbclient>=0.5.0 in /usr/local/lib/python3.10/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.8.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (23.1)\n",
            "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.10/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.5.0)\n",
            "Requirement already satisfied: tinycss2 in /usr/local/lib/python3.10/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.2.1)\n",
            "Requirement already satisfied: fastjsonschema in /usr/local/lib/python3.10/dist-packages (from nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (2.17.1)\n",
            "Requirement already satisfied: jsonschema>=2.6 in /usr/local/lib/python3.10/dist-packages (from nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (4.3.3)\n",
            "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=2.6->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (23.1.0)\n",
            "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=2.6->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.19.3)\n",
            "Requirement already satisfied: cffi>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from argon2-cffi-bindings->argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.15.1)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (2.4.1)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.10/dist-packages (from bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.5.1)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (2.21)\n",
            "Enabling notebook extension jupyter-js-widgets/extension...\n",
            "Paths used for configuration of notebook: \n",
            "    \t/root/.jupyter/nbconfig/notebook.json\n",
            "Paths used for configuration of notebook: \n",
            "    \t\n",
            "      - Validating: \u001b[32mOK\u001b[0m\n",
            "Paths used for configuration of notebook: \n",
            "    \t/root/.jupyter/nbconfig/notebook.json\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "%matplotlib inline\n",
        "\n",
        "from scipy.stats import rankdata\n",
        "\n",
        "import openml\n",
        "\n",
        "from sklearn.metrics import log_loss, accuracy_score, roc_auc_score, confusion_matrix\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.svm import SVC, LinearSVC\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from  sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
        "from sklearn.preprocessing import label_binarize, LabelEncoder\n",
        "\n",
        "import plotly.express as px\n",
        "\n",
        "\n",
        "from nonconformist.base import ClassifierAdapter\n",
        "from nonconformist.cp import IcpClassifier\n",
        "from nonconformist.nc import NcFactory, ClassifierNc, InverseProbabilityErrFunc, MarginErrFunc\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')"
      ],
      "metadata": {
        "id": "Kc-JdD9IsWgm"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "# Load dataset"
      ],
      "metadata": {
        "id": "uV4g10IOtW-6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# List datasets https://docs.openml.org/Python-API/\n",
        "datasets_df = openml.datasets.list_datasets(output_format=\"dataframe\")\n",
        "print(datasets_df.head(n=10))\n",
        "\n",
        "datasets_df.set_index('did', inplace = True)"
      ],
      "metadata": {
        "id": "XK2pin99tR1u",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f7412d73-7cc9-417e-9d83-59b655567cea"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "    did             name  version uploader  status format  MajorityClassSize  \\\n",
            "2     2           anneal        1        1  active   ARFF              684.0   \n",
            "3     3         kr-vs-kp        1        1  active   ARFF             1669.0   \n",
            "4     4            labor        1        1  active   ARFF               37.0   \n",
            "5     5       arrhythmia        1        1  active   ARFF              245.0   \n",
            "6     6           letter        1        1  active   ARFF              813.0   \n",
            "7     7        audiology        1        1  active   ARFF               57.0   \n",
            "8     8  liver-disorders        1        1  active   ARFF                NaN   \n",
            "9     9            autos        1        1  active   ARFF               67.0   \n",
            "10   10            lymph        1        1  active   ARFF               81.0   \n",
            "11   11    balance-scale        1        1  active   ARFF              288.0   \n",
            "\n",
            "    MaxNominalAttDistinctValues  MinorityClassSize  NumberOfClasses  \\\n",
            "2                           7.0                8.0              5.0   \n",
            "3                           3.0             1527.0              2.0   \n",
            "4                           3.0               20.0              2.0   \n",
            "5                          13.0                2.0             13.0   \n",
            "6                          26.0              734.0             26.0   \n",
            "7                          24.0                1.0             24.0   \n",
            "8                           NaN                NaN              0.0   \n",
            "9                          22.0                3.0              6.0   \n",
            "10                          8.0                2.0              4.0   \n",
            "11                          3.0               49.0              3.0   \n",
            "\n",
            "    NumberOfFeatures  NumberOfInstances  NumberOfInstancesWithMissingValues  \\\n",
            "2               39.0              898.0                               898.0   \n",
            "3               37.0             3196.0                                 0.0   \n",
            "4               17.0               57.0                                56.0   \n",
            "5              280.0              452.0                               384.0   \n",
            "6               17.0            20000.0                                 0.0   \n",
            "7               70.0              226.0                               222.0   \n",
            "8                6.0              345.0                                 0.0   \n",
            "9               26.0              205.0                                46.0   \n",
            "10              19.0              148.0                                 0.0   \n",
            "11               5.0              625.0                                 0.0   \n",
            "\n",
            "    NumberOfMissingValues  NumberOfNumericFeatures  NumberOfSymbolicFeatures  \n",
            "2                 22175.0                      6.0                      33.0  \n",
            "3                     0.0                      0.0                      37.0  \n",
            "4                   326.0                      8.0                       9.0  \n",
            "5                   408.0                    206.0                      74.0  \n",
            "6                     0.0                     16.0                       1.0  \n",
            "7                   317.0                      0.0                      70.0  \n",
            "8                     0.0                      6.0                       0.0  \n",
            "9                    59.0                     15.0                      11.0  \n",
            "10                    0.0                      3.0                      16.0  \n",
            "11                    0.0                      4.0                       1.0  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "datasets_df.head()"
      ],
      "metadata": {
        "id": "7vaAPOxQtcqM",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 299
        },
        "outputId": "76e1b687-8277-4c5b-d3f9-009f7ef82d3b"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "           name  version uploader  status format  MajorityClassSize  \\\n",
              "did                                                                   \n",
              "2        anneal        1        1  active   ARFF              684.0   \n",
              "3      kr-vs-kp        1        1  active   ARFF             1669.0   \n",
              "4         labor        1        1  active   ARFF               37.0   \n",
              "5    arrhythmia        1        1  active   ARFF              245.0   \n",
              "6        letter        1        1  active   ARFF              813.0   \n",
              "\n",
              "     MaxNominalAttDistinctValues  MinorityClassSize  NumberOfClasses  \\\n",
              "did                                                                    \n",
              "2                            7.0                8.0              5.0   \n",
              "3                            3.0             1527.0              2.0   \n",
              "4                            3.0               20.0              2.0   \n",
              "5                           13.0                2.0             13.0   \n",
              "6                           26.0              734.0             26.0   \n",
              "\n",
              "     NumberOfFeatures  NumberOfInstances  NumberOfInstancesWithMissingValues  \\\n",
              "did                                                                            \n",
              "2                39.0              898.0                               898.0   \n",
              "3                37.0             3196.0                                 0.0   \n",
              "4                17.0               57.0                                56.0   \n",
              "5               280.0              452.0                               384.0   \n",
              "6                17.0            20000.0                                 0.0   \n",
              "\n",
              "     NumberOfMissingValues  NumberOfNumericFeatures  NumberOfSymbolicFeatures  \n",
              "did                                                                            \n",
              "2                  22175.0                      6.0                      33.0  \n",
              "3                      0.0                      0.0                      37.0  \n",
              "4                    326.0                      8.0                       9.0  \n",
              "5                    408.0                    206.0                      74.0  \n",
              "6                      0.0                     16.0                       1.0  "
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-7c1fda98-18c5-44c8-aaad-8c9076022449\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>name</th>\n",
              "      <th>version</th>\n",
              "      <th>uploader</th>\n",
              "      <th>status</th>\n",
              "      <th>format</th>\n",
              "      <th>MajorityClassSize</th>\n",
              "      <th>MaxNominalAttDistinctValues</th>\n",
              "      <th>MinorityClassSize</th>\n",
              "      <th>NumberOfClasses</th>\n",
              "      <th>NumberOfFeatures</th>\n",
              "      <th>NumberOfInstances</th>\n",
              "      <th>NumberOfInstancesWithMissingValues</th>\n",
              "      <th>NumberOfMissingValues</th>\n",
              "      <th>NumberOfNumericFeatures</th>\n",
              "      <th>NumberOfSymbolicFeatures</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>did</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>anneal</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>active</td>\n",
              "      <td>ARFF</td>\n",
              "      <td>684.0</td>\n",
              "      <td>7.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>39.0</td>\n",
              "      <td>898.0</td>\n",
              "      <td>898.0</td>\n",
              "      <td>22175.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>33.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>kr-vs-kp</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>active</td>\n",
              "      <td>ARFF</td>\n",
              "      <td>1669.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>1527.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>37.0</td>\n",
              "      <td>3196.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>37.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>labor</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>active</td>\n",
              "      <td>ARFF</td>\n",
              "      <td>37.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>17.0</td>\n",
              "      <td>57.0</td>\n",
              "      <td>56.0</td>\n",
              "      <td>326.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>9.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>arrhythmia</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>active</td>\n",
              "      <td>ARFF</td>\n",
              "      <td>245.0</td>\n",
              "      <td>13.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>13.0</td>\n",
              "      <td>280.0</td>\n",
              "      <td>452.0</td>\n",
              "      <td>384.0</td>\n",
              "      <td>408.0</td>\n",
              "      <td>206.0</td>\n",
              "      <td>74.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>letter</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>active</td>\n",
              "      <td>ARFF</td>\n",
              "      <td>813.0</td>\n",
              "      <td>26.0</td>\n",
              "      <td>734.0</td>\n",
              "      <td>26.0</td>\n",
              "      <td>17.0</td>\n",
              "      <td>20000.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>16.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7c1fda98-18c5-44c8-aaad-8c9076022449')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-668654fc-7224-4399-bf9f-ad34e442d6fb\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-668654fc-7224-4399-bf9f-ad34e442d6fb')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-668654fc-7224-4399-bf9f-ad34e442d6fb button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-7c1fda98-18c5-44c8-aaad-8c9076022449 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-7c1fda98-18c5-44c8-aaad-8c9076022449');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "datasets_df.query('NumberOfClasses == 2')"
      ],
      "metadata": {
        "id": "AijKKhPmttli",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 616
        },
        "outputId": "183a521f-d36c-4512-8277-b597c6bc1a63"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                    name  version uploader  status format  \\\n",
              "did                                                                         \n",
              "3                               kr-vs-kp        1        1  active   ARFF   \n",
              "4                                  labor        1        1  active   ARFF   \n",
              "13                         breast-cancer        1        1  active   ARFF   \n",
              "15                              breast-w        1        1  active   ARFF   \n",
              "24                              mushroom        1        1  active   ARFF   \n",
              "...                                  ...      ...      ...     ...    ...   \n",
              "45568               telco-customer-churn        2       86  active   arff   \n",
              "45570                              Higgs        8       86  active   arff   \n",
              "45575                            Epsilon        1       86  active   arff   \n",
              "45577                Give-Me-Some-Credit        1       86  active   arff   \n",
              "45578  California-Housing-Classification        1       86  active   arff   \n",
              "\n",
              "       MajorityClassSize  MaxNominalAttDistinctValues  MinorityClassSize  \\\n",
              "did                                                                        \n",
              "3                 1669.0                          3.0             1527.0   \n",
              "4                   37.0                          3.0               20.0   \n",
              "13                 201.0                         11.0               85.0   \n",
              "15                 458.0                          2.0              241.0   \n",
              "24                4208.0                         12.0             3916.0   \n",
              "...                  ...                          ...                ...   \n",
              "45568             5174.0                          NaN             1869.0   \n",
              "45570          5829123.0                          NaN          5170877.0   \n",
              "45575           250222.0                          NaN           249778.0   \n",
              "45577           139974.0                          NaN            10026.0   \n",
              "45578            10323.0                          NaN            10317.0   \n",
              "\n",
              "       NumberOfClasses  NumberOfFeatures  NumberOfInstances  \\\n",
              "did                                                           \n",
              "3                  2.0              37.0             3196.0   \n",
              "4                  2.0              17.0               57.0   \n",
              "13                 2.0              10.0              286.0   \n",
              "15                 2.0              10.0              699.0   \n",
              "24                 2.0              23.0             8124.0   \n",
              "...                ...               ...                ...   \n",
              "45568              2.0              20.0             7043.0   \n",
              "45570              2.0              29.0         11000000.0   \n",
              "45575              2.0            2001.0           500000.0   \n",
              "45577              2.0              11.0           150000.0   \n",
              "45578              2.0               9.0            20640.0   \n",
              "\n",
              "       NumberOfInstancesWithMissingValues  NumberOfMissingValues  \\\n",
              "did                                                                \n",
              "3                                     0.0                    0.0   \n",
              "4                                    56.0                  326.0   \n",
              "13                                    9.0                    9.0   \n",
              "15                                   16.0                   16.0   \n",
              "24                                 2480.0                 2480.0   \n",
              "...                                   ...                    ...   \n",
              "45568                                11.0                   11.0   \n",
              "45570                                 0.0                    0.0   \n",
              "45575                                 0.0                    0.0   \n",
              "45577                             29731.0                33655.0   \n",
              "45578                                 0.0                    0.0   \n",
              "\n",
              "       NumberOfNumericFeatures  NumberOfSymbolicFeatures  \n",
              "did                                                       \n",
              "3                          0.0                      37.0  \n",
              "4                          8.0                       9.0  \n",
              "13                         0.0                      10.0  \n",
              "15                         9.0                       1.0  \n",
              "24                         0.0                      23.0  \n",
              "...                        ...                       ...  \n",
              "45568                      3.0                      17.0  \n",
              "45570                     28.0                       1.0  \n",
              "45575                   2000.0                       1.0  \n",
              "45577                     10.0                       1.0  \n",
              "45578                      8.0                       1.0  \n",
              "\n",
              "[1407 rows x 15 columns]"
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-c5bee5d2-6e0c-40eb-aaa8-521262e909d1\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>name</th>\n",
              "      <th>version</th>\n",
              "      <th>uploader</th>\n",
              "      <th>status</th>\n",
              "      <th>format</th>\n",
              "      <th>MajorityClassSize</th>\n",
              "      <th>MaxNominalAttDistinctValues</th>\n",
              "      <th>MinorityClassSize</th>\n",
              "      <th>NumberOfClasses</th>\n",
              "      <th>NumberOfFeatures</th>\n",
              "      <th>NumberOfInstances</th>\n",
              "      <th>NumberOfInstancesWithMissingValues</th>\n",
              "      <th>NumberOfMissingValues</th>\n",
              "      <th>NumberOfNumericFeatures</th>\n",
              "      <th>NumberOfSymbolicFeatures</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>did</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>kr-vs-kp</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>active</td>\n",
              "      <td>ARFF</td>\n",
              "      <td>1669.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>1527.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>37.0</td>\n",
              "      <td>3196.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>37.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>labor</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>active</td>\n",
              "      <td>ARFF</td>\n",
              "      <td>37.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>17.0</td>\n",
              "      <td>57.0</td>\n",
              "      <td>56.0</td>\n",
              "      <td>326.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>9.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>breast-cancer</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>active</td>\n",
              "      <td>ARFF</td>\n",
              "      <td>201.0</td>\n",
              "      <td>11.0</td>\n",
              "      <td>85.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>286.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>10.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>breast-w</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>active</td>\n",
              "      <td>ARFF</td>\n",
              "      <td>458.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>241.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>699.0</td>\n",
              "      <td>16.0</td>\n",
              "      <td>16.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>mushroom</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>active</td>\n",
              "      <td>ARFF</td>\n",
              "      <td>4208.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>3916.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>23.0</td>\n",
              "      <td>8124.0</td>\n",
              "      <td>2480.0</td>\n",
              "      <td>2480.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>23.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45568</th>\n",
              "      <td>telco-customer-churn</td>\n",
              "      <td>2</td>\n",
              "      <td>86</td>\n",
              "      <td>active</td>\n",
              "      <td>arff</td>\n",
              "      <td>5174.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1869.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>7043.0</td>\n",
              "      <td>11.0</td>\n",
              "      <td>11.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>17.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45570</th>\n",
              "      <td>Higgs</td>\n",
              "      <td>8</td>\n",
              "      <td>86</td>\n",
              "      <td>active</td>\n",
              "      <td>arff</td>\n",
              "      <td>5829123.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>5170877.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>11000000.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>28.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45575</th>\n",
              "      <td>Epsilon</td>\n",
              "      <td>1</td>\n",
              "      <td>86</td>\n",
              "      <td>active</td>\n",
              "      <td>arff</td>\n",
              "      <td>250222.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>249778.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2001.0</td>\n",
              "      <td>500000.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2000.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45577</th>\n",
              "      <td>Give-Me-Some-Credit</td>\n",
              "      <td>1</td>\n",
              "      <td>86</td>\n",
              "      <td>active</td>\n",
              "      <td>arff</td>\n",
              "      <td>139974.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>10026.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>11.0</td>\n",
              "      <td>150000.0</td>\n",
              "      <td>29731.0</td>\n",
              "      <td>33655.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45578</th>\n",
              "      <td>California-Housing-Classification</td>\n",
              "      <td>1</td>\n",
              "      <td>86</td>\n",
              "      <td>active</td>\n",
              "      <td>arff</td>\n",
              "      <td>10323.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>10317.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>20640.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1407 rows × 15 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c5bee5d2-6e0c-40eb-aaa8-521262e909d1')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-a9a2aff4-b144-4428-b282-78a25c94d8e8\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-a9a2aff4-b144-4428-b282-78a25c94d8e8')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-a9a2aff4-b144-4428-b282-78a25c94d8e8 button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-c5bee5d2-6e0c-40eb-aaa8-521262e909d1 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-c5bee5d2-6e0c-40eb-aaa8-521262e909d1');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# output information about selected dataset\n",
        "datasets_df.loc[1461]"
      ],
      "metadata": {
        "id": "rBVC3K7cuPZ9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "505d7a9a-409b-4012-a71f-34ac582370c3"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "name                                  bank-marketing\n",
              "version                                            1\n",
              "uploader                                          64\n",
              "status                                        active\n",
              "format                                          ARFF\n",
              "MajorityClassSize                            39922.0\n",
              "MaxNominalAttDistinctValues                     12.0\n",
              "MinorityClassSize                             5289.0\n",
              "NumberOfClasses                                  2.0\n",
              "NumberOfFeatures                                17.0\n",
              "NumberOfInstances                            45211.0\n",
              "NumberOfInstancesWithMissingValues               0.0\n",
              "NumberOfMissingValues                            0.0\n",
              "NumberOfNumericFeatures                          7.0\n",
              "NumberOfSymbolicFeatures                        10.0\n",
              "Name: 1461, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Bank marketing dataset https://www.openml.org/d/1461\n",
        "dataset = openml.datasets.get_dataset(1461)\n",
        "\n",
        "# Print a summary\n",
        "print(\n",
        "    f\"This is dataset '{dataset.name}', the target feature is \"\n",
        "    f\"'{dataset.default_target_attribute}'\"\n",
        ")\n",
        "print(f\"URL: {dataset.url}\")\n",
        "print(dataset.description[:500])"
      ],
      "metadata": {
        "id": "wANKvmG-u_NI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6f3cb267-0993-490b-92df-f41178feb008"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "This is dataset 'bank-marketing', the target feature is 'Class'\n",
            "URL: https://api.openml.org/data/v1/download/1586218/bank-marketing.arff\n",
            "**Author**: Paulo Cortez, Sérgio Moro\n",
            "**Source**: [UCI](https://archive.ics.uci.edu/ml/datasets/bank+marketing)\n",
            "**Please cite**: S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM'2011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.       \n",
            "\n",
            "**Bank Marketing**  \n",
            "The data is related with direct marketing campaigns of \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# openml API\n",
        "X, y, categorical_indicator, attribute_names = dataset.get_data(\n",
        "    dataset_format=\"array\", target=dataset.default_target_attribute\n",
        ")\n",
        "df = pd.DataFrame(X, columns=attribute_names)\n",
        "df[\"class\"] = y"
      ],
      "metadata": {
        "id": "ZQDjL66ovJCk"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "For more information, read [Moro et al., 2011].\n",
        "\n",
        "Input variables:\n",
        "\n",
        "bank client data:\n",
        "1 - age (numeric)\n",
        "\n",
        "2 - job : type of job (categorical: \"admin.\",\"unknown\",\"unemployed\",\"management\",\"housemaid\",\"entrepreneur\", \"student\",\"blue-collar\",\"self-employed\",\"retired\",\"technician\",\"services\")\n",
        "\n",
        "3 - marital : marital status (categorical: \"married\",\"divorced\",\"single\"; note: \"divorced\" means divorced or widowed)\n",
        "\n",
        "4 - education (categorical: \"unknown\",\"secondary\",\"primary\",\"tertiary\")\n",
        "\n",
        "5 - default: has credit in default? (binary: \"yes\",\"no\")\n",
        "\n",
        "6 - balance: average yearly balance, in euros (numeric)\n",
        "\n",
        "7 - housing: has housing loan? (binary: \"yes\",\"no\")\n",
        "\n",
        "8 - loan: has personal loan? (binary: \"yes\",\"no\")\n",
        "\n",
        "related with the last contact of the current campaign:\n",
        "9 - contact: contact communication type (categorical: \"unknown\",\"telephone\",\"cellular\")\n",
        "\n",
        "10 - day: last contact day of the month (numeric)\n",
        "\n",
        "11 - month: last contact month of year (categorical: \"jan\", \"feb\", \"mar\", ..., \"nov\", \"dec\")\n",
        "\n",
        "12 - duration: last contact duration, in seconds (numeric)\n",
        "\n",
        "other attributes:\n",
        "13 - campaign: number of contacts performed during this campaign and for this client (numeric, includes last contact)\n",
        "\n",
        "14 - pdays: number of days that passed by after the client was last contacted from a previous campaign (numeric, -1 means client was not previously contacted)\n",
        "\n",
        "15 - previous: number of contacts performed before this campaign and for this client (numeric)\n",
        "\n",
        "16 - poutcome: outcome of the previous marketing campaign (categorical: \"unknown\",\"other\",\"failure\",\"success\")\n",
        "\n",
        "output variable (desired target):\n",
        "17 - y - has the client subscribed a term deposit? (binary: \"yes\",\"no\")"
      ],
      "metadata": {
        "id": "jZN6xH3D-sCb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "categorical_indicator"
      ],
      "metadata": {
        "id": "wTAOXkSdoXnx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "30689aea-4a7c-4658-f1aa-0a2cc6ca99d0"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[False,\n",
              " True,\n",
              " True,\n",
              " True,\n",
              " True,\n",
              " False,\n",
              " True,\n",
              " True,\n",
              " True,\n",
              " False,\n",
              " True,\n",
              " False,\n",
              " False,\n",
              " False,\n",
              " False,\n",
              " True]"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "metadata": {
        "id": "g9iGKJn4vOT9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "e68b94a5-505d-44f0-dd51-c8cb1d782634"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     V1    V2   V3   V4   V5      V6   V7   V8   V9  V10  V11    V12  V13  \\\n",
              "0  58.0   4.0  1.0  2.0  0.0  2143.0  1.0  0.0  2.0  5.0  8.0  261.0  1.0   \n",
              "1  44.0   9.0  2.0  1.0  0.0    29.0  1.0  0.0  2.0  5.0  8.0  151.0  1.0   \n",
              "2  33.0   2.0  1.0  1.0  0.0     2.0  1.0  1.0  2.0  5.0  8.0   76.0  1.0   \n",
              "3  47.0   1.0  1.0  3.0  0.0  1506.0  1.0  0.0  2.0  5.0  8.0   92.0  1.0   \n",
              "4  33.0  11.0  2.0  3.0  0.0     1.0  0.0  0.0  2.0  5.0  8.0  198.0  1.0   \n",
              "\n",
              "   V14  V15  V16  class  \n",
              "0 -1.0  0.0  3.0      0  \n",
              "1 -1.0  0.0  3.0      0  \n",
              "2 -1.0  0.0  3.0      0  \n",
              "3 -1.0  0.0  3.0      0  \n",
              "4 -1.0  0.0  3.0      0  "
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-7e25ce22-9397-414f-a141-8007947c52ce\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>V1</th>\n",
              "      <th>V2</th>\n",
              "      <th>V3</th>\n",
              "      <th>V4</th>\n",
              "      <th>V5</th>\n",
              "      <th>V6</th>\n",
              "      <th>V7</th>\n",
              "      <th>V8</th>\n",
              "      <th>V9</th>\n",
              "      <th>V10</th>\n",
              "      <th>V11</th>\n",
              "      <th>V12</th>\n",
              "      <th>V13</th>\n",
              "      <th>V14</th>\n",
              "      <th>V15</th>\n",
              "      <th>V16</th>\n",
              "      <th>class</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>58.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2143.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>261.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>44.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>151.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>33.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>76.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>47.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1506.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>92.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>33.0</td>\n",
              "      <td>11.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>198.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7e25ce22-9397-414f-a141-8007947c52ce')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-767a9a5d-4f70-4579-9d01-a1f2624c1b58\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-767a9a5d-4f70-4579-9d01-a1f2624c1b58')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-767a9a5d-4f70-4579-9d01-a1f2624c1b58 button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-7e25ce22-9397-414f-a141-8007947c52ce button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-7e25ce22-9397-414f-a141-8007947c52ce');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for cat_ind, col in zip(categorical_indicator, df.columns):\n",
        "  print(cat_ind, col)\n",
        "  if cat_ind == True:\n",
        "    df[col] = df[col].astype('category')"
      ],
      "metadata": {
        "id": "Z5JFxVxNoevu",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f570478d-3ce1-4aec-8d03-97c5265afd06"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "False V1\n",
            "True V2\n",
            "True V3\n",
            "True V4\n",
            "True V5\n",
            "False V6\n",
            "True V7\n",
            "True V8\n",
            "True V9\n",
            "False V10\n",
            "True V11\n",
            "False V12\n",
            "False V13\n",
            "False V14\n",
            "False V15\n",
            "True V16\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.info()"
      ],
      "metadata": {
        "id": "mEm04QHynNOt",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b7950a2a-e9cd-408c-dff0-995f462ac020"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 45211 entries, 0 to 45210\n",
            "Data columns (total 17 columns):\n",
            " #   Column  Non-Null Count  Dtype   \n",
            "---  ------  --------------  -----   \n",
            " 0   V1      45211 non-null  float32 \n",
            " 1   V2      45211 non-null  category\n",
            " 2   V3      45211 non-null  category\n",
            " 3   V4      45211 non-null  category\n",
            " 4   V5      45211 non-null  category\n",
            " 5   V6      45211 non-null  float32 \n",
            " 6   V7      45211 non-null  category\n",
            " 7   V8      45211 non-null  category\n",
            " 8   V9      45211 non-null  category\n",
            " 9   V10     45211 non-null  float32 \n",
            " 10  V11     45211 non-null  category\n",
            " 11  V12     45211 non-null  float32 \n",
            " 12  V13     45211 non-null  float32 \n",
            " 13  V14     45211 non-null  float32 \n",
            " 14  V15     45211 non-null  float32 \n",
            " 15  V16     45211 non-null  category\n",
            " 16  class   45211 non-null  int64   \n",
            "dtypes: category(9), float32(7), int64(1)\n",
            "memory usage: 1.9 MB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['class'].value_counts()"
      ],
      "metadata": {
        "id": "Vt5W5pL0vZxk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "61f44596-d2de-4f43-b855-ee94262913cc"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    39922\n",
              "1     5289\n",
              "Name: class, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['class'].mean()"
      ],
      "metadata": {
        "id": "g3o9J-qIdZ-O",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ef8e9857-b253-4e29-e345-30eb6574af32"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.11698480458295547"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cat_columns = df.select_dtypes(\"category\").columns\n",
        "cat_columns"
      ],
      "metadata": {
        "id": "W9R2KwZOpzQG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "168c6cc2-f1a7-4d51-b178-031c13899a62"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['V2', 'V3', 'V4', 'V5', 'V7', 'V8', 'V9', 'V11', 'V16'], dtype='object')"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Encode categorical columns using Label Encoder\n",
        "le = LabelEncoder()\n",
        "\n",
        "for col in cat_columns:\n",
        "  df[col] = le.fit_transform(df[col])"
      ],
      "metadata": {
        "id": "O28bbJWuqbBZ"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.info()"
      ],
      "metadata": {
        "id": "9YDxAAxbquIf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dcc7898b-db0a-4271-a347-e359c7c42309"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 45211 entries, 0 to 45210\n",
            "Data columns (total 17 columns):\n",
            " #   Column  Non-Null Count  Dtype  \n",
            "---  ------  --------------  -----  \n",
            " 0   V1      45211 non-null  float32\n",
            " 1   V2      45211 non-null  int64  \n",
            " 2   V3      45211 non-null  int64  \n",
            " 3   V4      45211 non-null  int64  \n",
            " 4   V5      45211 non-null  int64  \n",
            " 5   V6      45211 non-null  float32\n",
            " 6   V7      45211 non-null  int64  \n",
            " 7   V8      45211 non-null  int64  \n",
            " 8   V9      45211 non-null  int64  \n",
            " 9   V10     45211 non-null  float32\n",
            " 10  V11     45211 non-null  int64  \n",
            " 11  V12     45211 non-null  float32\n",
            " 12  V13     45211 non-null  float32\n",
            " 13  V14     45211 non-null  float32\n",
            " 14  V15     45211 non-null  float32\n",
            " 15  V16     45211 non-null  int64  \n",
            " 16  class   45211 non-null  int64  \n",
            "dtypes: float32(7), int64(10)\n",
            "memory usage: 4.7 MB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Split into training and test set"
      ],
      "metadata": {
        "id": "mRQ5oestwZXo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "X = df.drop('class', axis = 1)\n",
        "y = df['class']"
      ],
      "metadata": {
        "id": "VKqATqO5wyUC"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X.head()"
      ],
      "metadata": {
        "id": "GHcOiGT9xEAU",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "04e9b8c6-50ef-4a97-9ef4-81bfe45f61e2"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     V1  V2  V3  V4  V5      V6  V7  V8  V9  V10  V11    V12  V13  V14  V15  \\\n",
              "0  58.0   4   1   2   0  2143.0   1   0   2  5.0    8  261.0  1.0 -1.0  0.0   \n",
              "1  44.0   9   2   1   0    29.0   1   0   2  5.0    8  151.0  1.0 -1.0  0.0   \n",
              "2  33.0   2   1   1   0     2.0   1   1   2  5.0    8   76.0  1.0 -1.0  0.0   \n",
              "3  47.0   1   1   3   0  1506.0   1   0   2  5.0    8   92.0  1.0 -1.0  0.0   \n",
              "4  33.0  11   2   3   0     1.0   0   0   2  5.0    8  198.0  1.0 -1.0  0.0   \n",
              "\n",
              "   V16  \n",
              "0    3  \n",
              "1    3  \n",
              "2    3  \n",
              "3    3  \n",
              "4    3  "
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-505968bb-4eac-438b-ab94-ae8cf4dedb3c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>V1</th>\n",
              "      <th>V2</th>\n",
              "      <th>V3</th>\n",
              "      <th>V4</th>\n",
              "      <th>V5</th>\n",
              "      <th>V6</th>\n",
              "      <th>V7</th>\n",
              "      <th>V8</th>\n",
              "      <th>V9</th>\n",
              "      <th>V10</th>\n",
              "      <th>V11</th>\n",
              "      <th>V12</th>\n",
              "      <th>V13</th>\n",
              "      <th>V14</th>\n",
              "      <th>V15</th>\n",
              "      <th>V16</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>58.0</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>2143.0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>5.0</td>\n",
              "      <td>8</td>\n",
              "      <td>261.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>44.0</td>\n",
              "      <td>9</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>5.0</td>\n",
              "      <td>8</td>\n",
              "      <td>151.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>33.0</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>5.0</td>\n",
              "      <td>8</td>\n",
              "      <td>76.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>47.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>1506.0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>5.0</td>\n",
              "      <td>8</td>\n",
              "      <td>92.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>33.0</td>\n",
              "      <td>11</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>5.0</td>\n",
              "      <td>8</td>\n",
              "      <td>198.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-505968bb-4eac-438b-ab94-ae8cf4dedb3c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-2a704e60-bea8-4282-ab33-04acf738a68c\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-2a704e60-bea8-4282-ab33-04acf738a68c')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-2a704e60-bea8-4282-ab33-04acf738a68c button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-505968bb-4eac-438b-ab94-ae8cf4dedb3c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-505968bb-4eac-438b-ab94-ae8cf4dedb3c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y.head()"
      ],
      "metadata": {
        "id": "fHxKB3j9xE9Q",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6972c336-753f-4735-c9e6-a8297616d377"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    0\n",
              "1    0\n",
              "2    0\n",
              "3    0\n",
              "4    0\n",
              "Name: class, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "X_train_calib, X_test, y_train_calib, y_test = train_test_split(X, y, test_size=1000, random_state=42, stratify=y)\n",
        "\n",
        "X_train, X_calib, y_train, y_calib = train_test_split(X_train_calib, y_train_calib, test_size=1000, random_state=42, stratify=y_train_calib)"
      ],
      "metadata": {
        "id": "i_yvdFRLxa32"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Training set size {}'.format(len(y_train)))\n",
        "print('Calibration set size {}'.format(len(y_calib)))\n",
        "print('Test set size {}'.format(len(y_test)))"
      ],
      "metadata": {
        "id": "PPsHPWmB_KfW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5a5fb6a3-4f2f-4ea4-ea1c-30d4c67bb077"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training set size 43211\n",
            "Calibration set size 1000\n",
            "Test set size 1000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Train Logistic Regression"
      ],
      "metadata": {
        "id": "VflFBDa7zLKY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Train a logistic regression model\n",
        "model = LogisticRegression()\n",
        "model.fit(X_train, y_train)"
      ],
      "metadata": {
        "id": "NbFFrBm7zdAW",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        },
        "outputId": "17412c9d-4eb2-4c1c-8aff-7ae99e48504d"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LogisticRegression()"
            ],
            "text/html": [
              "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LogisticRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression()</pre></div></div></div></div></div>"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# make point predictions and obtain class scores on the calibration and test sets\n",
        "y_pred_calib = model.predict(X_calib)\n",
        "y_pred_score_calib = model.predict_proba(X_calib)\n",
        "\n",
        "y_pred_test = model.predict(X_test)\n",
        "y_pred_score_test = model.predict_proba(X_test)"
      ],
      "metadata": {
        "id": "pZfEuSWfzhES"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Classification accuracy on the test: {}'.format(accuracy_score(y_test, y_pred_test)))"
      ],
      "metadata": {
        "id": "YiZb5DSAzq_S",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7d7a354e-4b17-4f6c-f6bc-b862448fa15c"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Classification accuracy on the test: 0.887\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('ROC AUC on the test set: {}'.format(roc_auc_score(y_test, y_pred_score_test[:,1])))"
      ],
      "metadata": {
        "id": "aLCDuZky0YUE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2c514c9c-86b4-4ef4-a167-3528c9d4b627"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ROC AUC on the test set: 0.7820851603411059\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# predict classification scores on the training set\n",
        "y_pred_score_train = model.predict_proba(X_train)"
      ],
      "metadata": {
        "id": "ZxaolziZj_Io"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Define non-conformity measure"
      ],
      "metadata": {
        "id": "A6YfDAWqjXCB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# We use inverse probability nonconformity measure (also known as hinge loss). The function below calculates the probability of not predicting the correct class by simply looking up\n",
        "# probability score predicted by the underlying model for the correct class and subtracting it from 1.\n",
        "# For each correct output in y, inverse probablity nonconformity (hinge loss) score  is defined as math:: 1 - \\hat{P}(y_i | x)\n",
        "# This measure is also known as a hinge loss and is based simply on the probability estimate provided by the underlying point classification model for the correct class label y_i\n",
        "\n",
        "def InverseProbabilityNC(predicted_score, y):\n",
        "  prob = np.zeros(y.size, dtype=np.float32)\n",
        "  for i, y_ in enumerate(y):\n",
        "    if y_ >= predicted_score.shape[1]:\n",
        "      prob[i] = 0\n",
        "    else:\n",
        "      prob[i] = predicted_score[i, int(y_)]\n",
        "  return 1 - prob"
      ],
      "metadata": {
        "id": "GfxfX-xYjVww"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Let us see how Inductive Conformal Prediction works using the first point from the test dataset\n",
        "pd.DataFrame(X_test.iloc[0]).T"
      ],
      "metadata": {
        "id": "hFJZHTmojiXI",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        },
        "outputId": "43a873ba-e4f6-4afe-f43d-f62731388fda"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         V1   V2   V3   V4   V5      V6   V7   V8   V9  V10  V11   V12  V13  \\\n",
              "34262  41.0  0.0  1.0  2.0  0.0  1536.0  0.0  0.0  0.0  4.0  8.0  54.0  2.0   \n",
              "\n",
              "       V14  V15  V16  \n",
              "34262 -1.0  0.0  3.0  "
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-bcb6b375-3e89-41e4-83f5-a20b7f4c2fea\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>V1</th>\n",
              "      <th>V2</th>\n",
              "      <th>V3</th>\n",
              "      <th>V4</th>\n",
              "      <th>V5</th>\n",
              "      <th>V6</th>\n",
              "      <th>V7</th>\n",
              "      <th>V8</th>\n",
              "      <th>V9</th>\n",
              "      <th>V10</th>\n",
              "      <th>V11</th>\n",
              "      <th>V12</th>\n",
              "      <th>V13</th>\n",
              "      <th>V14</th>\n",
              "      <th>V15</th>\n",
              "      <th>V16</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>34262</th>\n",
              "      <td>41.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1536.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>54.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>-1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-bcb6b375-3e89-41e4-83f5-a20b7f4c2fea')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-6a10e249-f8d8-4b9e-84fd-73cd8c7dd00f\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-6a10e249-f8d8-4b9e-84fd-73cd8c7dd00f')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-6a10e249-f8d8-4b9e-84fd-73cd8c7dd00f button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-bcb6b375-3e89-41e4-83f5-a20b7f4c2fea button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-bcb6b375-3e89-41e4-83f5-a20b7f4c2fea');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y_pred_score_train"
      ],
      "metadata": {
        "id": "wtqOWPuNjiKN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "64bbae08-e2ad-431d-9ad1-c67cc2dfbeb4"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.90818872, 0.09181128],\n",
              "       [0.76816794, 0.23183206],\n",
              "       [0.86245949, 0.13754051],\n",
              "       ...,\n",
              "       [0.94751381, 0.05248619],\n",
              "       [0.898834  , 0.101166  ],\n",
              "       [0.95412253, 0.04587747]])"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y_train"
      ],
      "metadata": {
        "id": "dPyuCRVXjiEQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "365f64a8-d6ff-4857-8397-f7af96ff06ff"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "42042    1\n",
              "33646    0\n",
              "39810    0\n",
              "38929    0\n",
              "11706    0\n",
              "        ..\n",
              "39295    0\n",
              "35497    0\n",
              "24445    0\n",
              "19573    0\n",
              "831      0\n",
              "Name: class, Length: 43211, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x = X_test.iloc[0]\n",
        "x"
      ],
      "metadata": {
        "id": "YVwuWdxJjh-T",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e5ed748f-fb0d-4e07-d924-149b262afc41"
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "V1       41.0\n",
              "V2        0.0\n",
              "V3        1.0\n",
              "V4        2.0\n",
              "V5        0.0\n",
              "V6     1536.0\n",
              "V7        0.0\n",
              "V8        0.0\n",
              "V9        0.0\n",
              "V10       4.0\n",
              "V11       8.0\n",
              "V12      54.0\n",
              "V13       2.0\n",
              "V14      -1.0\n",
              "V15       0.0\n",
              "V16       3.0\n",
              "Name: 34262, dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y = y_test.iloc[0]\n",
        "y"
      ],
      "metadata": {
        "id": "joh95h2Cjh4l",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "28d3af39-c246-48f4-87f3-f9ebe125af55"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# function computes p-values for the test set using classical definition as per Vovk's book by essentially comparing the nonconformity score of the new test point\n",
        "# with the nonconformity scores of the points in the training set and computing ratio (with test point added to the same 'bag' as training points) of the # of training points\n",
        "# that have nonconformity scores greater than equal to that of the new test point\n",
        "\n",
        "def compute_p_values(non_conformity_scores_calibration, non_conformity_scores_test):\n",
        "  p_values = []\n",
        "  if not isinstance(non_conformity_scores_test, list):\n",
        "    non_conformity_scores_test = [non_conformity_scores_test]\n",
        "\n",
        "  for i, test_point_conformity_score in enumerate(non_conformity_scores_test):\n",
        "    p_value = (np.sum(non_conformity_scores_calibration >= test_point_conformity_score) + 1) / (len(non_conformity_scores_calibration)+1)\n",
        "    p_values.append(p_value)\n",
        "  return np.array(p_values)"
      ],
      "metadata": {
        "id": "P9UQc02Ul96i"
      },
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "y_pred_score_calib"
      ],
      "metadata": {
        "id": "BYQWI5EjmLph",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0ce37156-5708-4d2d-dae6-144905548730"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.89169458, 0.10830542],\n",
              "       [0.81965644, 0.18034356],\n",
              "       [0.77322923, 0.22677077],\n",
              "       ...,\n",
              "       [0.95855771, 0.04144229],\n",
              "       [0.86589232, 0.13410768],\n",
              "       [0.95778027, 0.04221973]])"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "1-InverseProbabilityNC(y_pred_score_calib,y_calib)[:20]"
      ],
      "metadata": {
        "id": "LJG2LpKrmWvH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b357148a-dfe5-4dff-cc27-baeaaa1b47a3"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.8916946 , 0.81965643, 0.77322924, 0.97244287, 0.884356  ,\n",
              "       0.63385993, 0.10243326, 0.8409457 , 0.05721056, 0.35713017,\n",
              "       0.6804593 , 0.761346  , 0.94911814, 0.98009074, 0.69222385,\n",
              "       0.96853656, 0.9542381 , 0.9299109 , 0.94860786, 0.9894122 ],\n",
              "      dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "non_conformity_scores_calibration = InverseProbabilityNC(y_pred_score_calib,y_calib)\n",
        "non_conformity_scores_calibration[:20]"
      ],
      "metadata": {
        "id": "g7txyDuyncXK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6cf32a31-43be-4dd8-c613-cf7b2395d61c"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.10830539, 0.18034357, 0.22677076, 0.02755713, 0.11564398,\n",
              "       0.36614007, 0.89756674, 0.15905428, 0.94278944, 0.64286983,\n",
              "       0.31954068, 0.23865402, 0.05088186, 0.01990926, 0.30777615,\n",
              "       0.03146344, 0.04576188, 0.0700891 , 0.05139214, 0.01058781],\n",
              "      dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y_pred_score_test"
      ],
      "metadata": {
        "id": "_oK5tVwWoZJx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e3b3be25-1ef3-4bf3-cb37-304ac5fa85b6"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.94627332, 0.05372668],\n",
              "       [0.94555113, 0.05444887],\n",
              "       [0.96514206, 0.03485794],\n",
              "       ...,\n",
              "       [0.92455361, 0.07544639],\n",
              "       [0.82048862, 0.17951138],\n",
              "       [0.7454973 , 0.2545027 ]])"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "non_conformity_scores_y_test_class_0 = 1 - y_pred_score_test[0,0]\n",
        "non_conformity_scores_y_test_class_1 = 1 - y_pred_score_test[0,1]"
      ],
      "metadata": {
        "id": "sNMgmWRapfuh"
      },
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "non_conformity_scores_calib_plus_test_0 = np.append(non_conformity_scores_calibration, non_conformity_scores_y_test_class_0)\n",
        "non_conformity_scores_calib_plus_test_1 = np.append(non_conformity_scores_calibration, non_conformity_scores_y_test_class_1)"
      ],
      "metadata": {
        "id": "w2K_yXj0q_Kj"
      },
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# compute p-value of the test point twice, each time assigning potential label 0 and 1\n",
        "p_value_class_0 = compute_p_values(non_conformity_scores_calib_plus_test_0, non_conformity_scores_y_test_class_0)\n",
        "p_value_class_1 = compute_p_values(non_conformity_scores_calib_plus_test_0, non_conformity_scores_y_test_class_1)"
      ],
      "metadata": {
        "id": "zgZq1gDFjhRv"
      },
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "p_value_class_0"
      ],
      "metadata": {
        "id": "PEBM7yRZjgjk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bb6c3870-0225-44af-875e-97522254b9ee"
      },
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.64670659])"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "p_value_class_1"
      ],
      "metadata": {
        "id": "u7bySdtBrXpb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d4b67d59-a765-4910-dc46-093b7f24d8f1"
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.00898204])"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('p-value for class 0: {}, p-value for class 1: {}'.format(p_value_class_0,p_value_class_1))"
      ],
      "metadata": {
        "id": "sYZBb4dUreR5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1d04b6bf-3a37-425d-fabf-0f8f5ec7a01c"
      },
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "p-value for class 0: [0.64670659], p-value for class 1: [0.00898204]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# plot distribution of nonconformity score for both situations\n",
        "\n",
        "plt.figure(figsize=(12, 8), dpi=80)\n",
        "plt.hist(np.sort(non_conformity_scores_calibration), label='Calibration set', alpha=0.6)\n",
        "# single line\n",
        "plt.vlines(x = non_conformity_scores_y_test_class_0, ymin = 0, ymax = 175, colors = 'green',\n",
        "           label = 'Nonconformity score for Class 0')\n",
        "\n",
        "plt.vlines(x = non_conformity_scores_y_test_class_1, ymin = 0, ymax = 175, colors = 'red',\n",
        "           label = 'Nonconformity score for Class 1')\n",
        "\n",
        "plt.grid(True)\n",
        "plt.ylabel('Nonconformity value')\n",
        "plt.title('Distribution of nonconformity values')\n",
        "\n",
        "plt.legend( loc='upper right')\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "SKB5n6-IreHY",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 560
        },
        "outputId": "58da02f8-aa35-474a-d522-5ebad988daec"
      },
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 960x640 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAyMAAAIfCAYAAABjHAOnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAxOAAAMTgF/d4wjAABNkElEQVR4nO3deVyVZf7/8fcBVFQWHdxFQARNAUXUQjOXXDBL/Y1LUy5FiaKNOY7W6FSmpllmkWVTYqhlpGXpoGMLSmrlNxyz0NIaFQVxHUwFV2S7f3/49Xwj9HhED/fp+Ho+Hj6Gc1/3fd2fc3PRnPe57sViGIYhAAAAAKhkbmYXAAAAAODWRBgBAAAAYArCCAAAAABTEEYAAAAAmIIwAgAAAMAUhBEAAAAApiCMAAAAADAFYQSAS5o9e7Z69+590/sNCwvTu+++K0nKzs6WxWJRZmbmTd/PPffco5kzZ970fq9HUVGRhg0bpj/84Q/y8vJSfn6+qfU40vr169WiRQt5e3tr8uTJlbrvMWPGKC4urlL3eS2bNm2SxWJRcXGx2aUAcHEWHnoI4PekW7du+uabb1StWjW5ubnJx8dH7dq106hRo3TvvfdWqM+goCA988wz1/2BMDs7W02bNtXevXsVEhJSoX3fjD4cZfny5Zo0aZL+85//yMfHx+xyHKpFixaKi4vTk08+aXYpFR6PN9OmTZvUvXt3FRUVycPDw7Q6ALg+ZkYA/O787W9/05kzZ5Sfn6/vvvtOvXv31gMPPKCnn37aYfssLCx0WN/Oat++fQoODnb5ICJdeq9t27at8PaGYTCLAAAVQBgB8LtWr149PfbYY5o3b55efPFF6ylT06dPV+fOna3rvfHGG2rWrJm8vb1Vv359xcbGSrp0OlROTo7GjRsnLy8vhYWFldl+6tSpatSokSIjIyVd+tY6KSmpTA0bN25Uq1at5OPjox49emjfvn3Wtm7duumZZ54ps/6v+7i8vzZt2sjLy0tjxoy54nY///yz7rnnHtWpU0f+/v6Kj48vc9pUt27d9Je//EVDhw6Vr6+vmjRporfeesvmsbPVZ2xsrJ577jmlp6fLy8tL99xzzxX7iI2N1QMPPKBx48bJz89P9evX19SpU8usk56errvuuku1a9dW06ZNNWXKFF28eLHM8XjuuefUt29feXt7q1mzZvrnP/9Zpo9PPvlE0dHRql27tvz8/DR48OCbcmz27NkjLy8vlZSUqF+/fvLy8tLXX38tSXrnnXcUHh4uHx8fhYeHW0/Pk/7vFL1FixapTZs2qlGjhrZt26Zu3brp8ccf15/+9Cf5+PjI399fH3zwgX788Ud17NhR3t7euv3227V79+4yx3D48OGSrjweMzMz5eHhoezs7DLH5L777tOECRPK/U7y8/NVo0YN6/u4bPz48erfv7+kSzMfnTp1kp+fn2rXrq27775b27dvv+Lv+Lc1/vq4/nqMHj58WEOHDlXjxo1Vr149Pfjggzp+/Li1/Wp/gwBucQYA/I507drVePrpp8stP3/+vOHm5mYsWLDAMAzDmDZtmnHnnXcahmEYe/bsMapXr278+OOPhmEYxpkzZ4wvv/zSum1gYKDx9ttvl+lv2rRphru7uzFjxgzjwoULxrlz58qtm5WVZUgyOnToYOTk5Bjnzp0zRo4cabRs2dIoKiq6ar1X6mPv3r1XfZ+nT582GjVqZEycONE4d+6cceTIEaNLly7GgAEDyqzv4+NjfPHFF0ZJSYnx8ccfG25ubuX6vcyePn99DK/m4YcfNqpWrWosX77cKC4uNr755hvDw8PD2LBhg2EYhnHgwAGjRo0axquvvmpcvHjR2LNnj9GqVSvjL3/5S5nj0aRJE+O7774zSkpKjFdeecXw9vY28vPzDcMwjHXr1hmenp7GihUrjIsXLxoXLlww0tLSbuqxkWSsX7/e+vrjjz82vL29jbS0NKO4uNhYv369UbNmTeOf//xnmd9bp06djJycHKO4uNgoKCgwunbtavj6+hpffvmlUVJSYsybN8+oUaOG0bdvXyMrK8u4ePGiMXDgQKN3795ljuGwYcPKHI/fjsdevXoZTz31lPX1gQMHDHd3d+Pnn3++4u9lxIgRxsMPP2x9feHCBaN27drG6tWrDcMwjM2bNxv/8z//Y1y8eNE4ffq0MWrUKCMgIMC4ePGiYRiGsXHjRkOSdRz/tsbLx/XyGC0oKDBatGhhTJo0yTh79qxx5swZY/jw4UbPnj0Nw7j23yCAWxczIwBcQvXq1VWnTh2dOHGiXJuHh4cMw9CuXbt0+vRpeXl5qUuXLtfs8/K3/J6enqpRo8ZV15s6daqaNGmiGjVq6NVXX9WePXuUnp5+Q+/n19auXavCwkLNmTNHNWrUUMOGDTVv3jytXr1ax44ds643aNAg3X333XJzc9OgQYP0hz/8Qd99990N9WmPO++8Uw888IDc3d3VsWNHRUZGauvWrZKk999/X82bN9eECRNUtWpVhYaGatasWVq4cKGMX12yGBcXp6ioKLm5uSk+Pl5nzpzRzz//LEl67bXXNHLkSA0ZMkRVq1aVp6enevTo4bBjI0kLFy7UyJEj1aNHD7m7u6tnz54aOXKkFixYUGa9Z599Vk2aNJG7u7uqVatm3VeXLl3k5uam2NhYnT9/XsOHD1dQUJCqVq2qoUOHWo+PvcaOHaslS5ZYTwVLSkpS586dddttt11x/bi4OH300Uc6ffq0JGnlypXy9PS0Xld15513qlOnTqpataq8vb01Z84c5eTklJmxuR6ffPKJzpw5o7lz56pmzZry8vLSiy++qLS0NB06dKjCf4MAXB9hBIBLuHDhgo4fPy4/P79ybU2bNtUHH3ygJUuWKCAgQB06dNDy5cuv2WdgYKAsFss112vatKn1Z29vb9WpU0cHDx68vjdgw8GDBxUYGFjmQuLLF7vn5ORYlzVq1KjMdjVr1tSZM2duqE972NrvwYMH1axZszLtISEh1t/XlfqoWbOmJFn7yMrKUosWLW7ofVzPsbFV92+Pza9/95c1bNiw3Hv57TJb+76S/v37y93dXWvWrFFxcbEWLVqk+Pj4q67fpUsX+fv7W8d5UlKSYmNj5e7uLkn64Ycf1K9fPzVu3Fg+Pj7W95Gbm3tddV22d+9e/fe//1Xt2rVVq1Yt1apVS2FhYapWrZpycnIq/DcIwPURRgC4hOXLl8tisejuu+++YvuAAQP0+eef65dfftGTTz6pYcOGac+ePZIkN7cr/6fwast/69fn8p89e1a//PKL/P39JV0KJ+fOnbO2FxcXl/nAZ88+mjRpopycnDIXSF++LiUgIMCuGiujz6vtZ//+/WWW7du3T9WrV1fdunXt6iMoKMj6u7pS/454H02aNClz7c/lfn/bp71j5HpcqU93d3eNGjVKCxcu1Nq1a1VUVKRBgwbZ7GfkyJFKSkpSZmamvvrqK40cOdLaNmTIEDVr1kw7d+7U6dOnlZWVJUllZqt+7bfjWJKOHDli/blBgwYKDAxUXl5emX8FBQXq1KmTJNt/gwBuXYQRAL9rx48fV2JioiZMmKAnn3xSoaGh5dbZvXu3Pv30U509e1YeHh7y9fWVJOu3xA0aNKjw6SmSNGvWLB06dEjnz5/XpEmTFBISYv0A1r59e61Zs0ZHjhzRhQsXNGXKFBUVFVm3rVu3rtzc3Gzu/95775WHh4eeeuopXbhwQceOHdNf//pX9evXTw0aNKhQzY7o80qGDh2q3bt3a/78+SosLNS+ffs0depUxcXF2TXrJEl/+ctftGjRIq1cuVKFhYUqKCjQF1984dD3ERcXp8WLF2vTpk0qKSnRhg0btGjRIo0ePbrCfdrrauNx1KhR2rhxo2bMmKHY2FhVrVrVZj8PP/ywduzYob/+9a/q2rVrmZme/Px8+fj4yNfXVydPntSkSZNs9tW+fXtt3LhR//nPf1RUVKR58+ZZA4wkDRw4UEVFRZo6dar15gG5ubn68MMPJV37bxDArYswAuB356WXXpKXl5d8fHzUtm1bffrpp0pOTtaLL754xfULCwv1/PPPW09JmTRpkpYuXWr9cPbss89q9erVqlWrllq3bn3d9YwcOVK9evVS/fr1tWfPHv3rX/+ynjb017/+Ve3atVPLli3VokULhYSEqHHjxtZtq1evrtmzZysuLk61atXSY489Vq5/Hx8frV+/Xjt27JC/v7/atWunkJCQMnd3ul6O6PNKAgMDtW7dOn344YeqV6+e7r77bt1zzz166aWX7O6jd+/eWr58uV588UXVrVtX/v7+SkxMdOj7GDJkiF555RU99thjqlWrlh5//HG99tprGjhw4A31a4+rjceGDRuqf//+2rFjh81TtC6rX7++7rvvPq1du7bcM0sWL16sjz76SN7e3oqOjr7q3dIuGzZsmB544AF16tRJTZo0UV5enu68805ru7e3t9LT05WTk6OIiAj5+PioU6dO+uqrryRd+28QwK2Lhx4CAPA7MXnyZH3//fdav3692aUAwE3BY1UBAPgdyMnJ0aJFi/Tee++ZXQoA3DScpgUAgJMbNmyYWrVqpaFDh17zlCoA+D3hNC0AAAAApmBmBAAAAIApCCMAAAAATOFSF7BXq1bN7odoOdLFixdVrVo1s8uAk2J8wBbGB66GsQFbGB+wxczxcfz4cV28ePGq7S4VRurWratDhw6ZXYZSU1MVExNjdhlwUowP2ML4wNUwNmAL4wO2mDk+/P39bbY7/DStixcvaty4cQoNDVVERISGDx8uSdq7d686deqk5s2bq0OHDtq1a5d1G1ttAAAAAFyDw8PIlClTZLFYtGfPHv344496+eWXJUnx8fEaPXq09uzZo8mTJys2Nta6ja02AAAAAK7BoWHk3LlzWrRokZ5//nlZLBZJUoMGDZSbm6tt27ZZZ0kGDRqkgwcPKjMz02YbAAAAANfh0GtG9u3bpz/84Q+aPXu20tLSVL16dU2fPl21atVSw4YN5eFxafcWi0UBAQHKycmRr6/vVdtCQkIcWS4AAHBxpaWlctVHrJWUlJhdApyYI8eHxWKRm1vF5jgcGkaKi4t14MABtWrVSi+++KIyMjLUq1cvffLJJzel/4SEBCUkJFhf5+XlKTU19ab0fSMKCgqcog44J8YHbGF84GoYGzfGzc1Nvr6+1i87XU2dOnW0Y8cOs8uAk6qM8VFcXKz8/HyVlpZe13YOfQL7L7/8ovr166uwsFDu7u6SpA4dOujJJ59UXFycTp48KQ8PDxmGoYYNG2rz5s3y8fFRSEjIFduuNTPi7+/P3bTg9BgfsIXxgathbNyYzMxMeXt7y8/Pz3rquCs5ffq0fHx8zC4DTsrR48MwDJ04cUJnzpwp93n9Wp/PHfr1QJ06ddSjRw+lpqaqb9++ysrKUlZWlu68805FRUUpOTlZsbGxWrlypfz9/a3F22oDAAC4HqWlpSoqKpKfn5/Lzoy4ublZv/gFfqsyxoefn59Onjyp0tLS6zply+F/kQsWLNDIkSM1efJkubm5KTExUY0bN1ZiYqJiY2M1e/Zs+fj4aMmSJdZtbLUBAABcj8sngbjijAjgLC7/fV3vSVcODyPBwcHauHFjueUtWrRQenr6Fbex1QYAAADANbjmXCUAAIANf1/1o8P6fmFgxDXXKS4u1vPPP6/ly5fLw8NDHh4euv322/XSSy+pVq1aV93unXfeUUpKilJSUrRt2zbNnTtXH374oQ4cOKAuXbooLy/vhutPSUlRgwYNFB0dLUll9mOWvLw8LViwQFOmTDGtBjiGwx96CAAAgLJGjhypbdu2KT09XTt37rTecfTkyZN299G+ffsKBYTi4mKb7SkpKdqyZcsN7+dmysvL04svvmhqDXAMwggAAEAlyszM1EcffaQlS5aodu3aki6dbz9kyBAFBwfr2LFj6t69u9q1a6ewsDCNGzfuirdL3bRpkyIjI8sse+KJJ9S6dWuFhYUpLS1NkpSdna1atWpp8uTJioqK0htvvKEvvvhCHTt2VNu2bRUWFqZFixZJkj799FOtWbNGc+fOVWRkpJKSksrt57333lPr1q3VunVr3XvvvTp8+LCkS7M2PXv21IMPPqiIiAi1b99e+/fvv+IxmDVrllq2bKnIyEhFRkbqwIEDkqRvv/1Wd999t9q3b6+2bdvqo48+kiSNGTNGZ86cUWRkpNq3b1/xgw+nw2laAAAAlej7779XaGio6tSpc8X2WrVq6V//+pe8vLxUUlKiAQMGaMWKFXrggQds9pufn6+WLVvq5Zdf1pYtW9S/f3/t27fP2hYWFqY5c+ZIkk6dOqXNmzfL3d1dJ0+eVNu2bRUTE6O+ffuqf//+ioyM1IQJEyRdCj2X7dy5U08++aS+++47NW7cWM8//7zi4uL02WefSboUJrZv366mTZtqypQpmjNnjhITE8vUeerUKb388ss6evSoqlevrvPnz8vNzU15eXkaPXq0Pv30UzVs2FC//PKLoqKi1KlTJy1YsECRkZHavn17BY44nBkzIwAAAE6ktLRUkydPVps2bdS2bVtt27bNrg/hHh4eio2NlSRFR0erUaNGysjIkCRVqVJFw4cPt6574sQJDRkyROHh4br77rt14sQJ7dy585r72Lhxo/r06aPGjRtLkh577DFt2LDB+nTvjh07qmnTptafL4ehX/Px8VFoaKiGDx+uxMREnTx5Up6envrmm2+0f/9+3XPPPYqMjFTPnj0lSbt3775mXfj9YmYEAACgEkVFRWnv3r06ceKE/Pz8yrUnJCQoNzdX//73v+Xp6amJEyeqoKCgQvu6fLvVGjVqlHn2w5gxY9S3b1+tXLlSFotFUVFRFdrHb2+X7Onpaf3Z3d39itenuLu7a8uWLfrmm2+0adMmRUdHa/ny5TIMQ2FhYfrmm2/KbZOdnX3dteH3gZkRAACAShQSEqJBgwZp5MiR1rtfGYahlStXav/+/Tp16pQaNGggT09PHTt2zHrdxLUUFxfrvffekyRt3bpVR44cKXdNyWWnTp1SYGCgLBaLvvrqK+3YscPa5uPjo/z8/Ctu1717d33++ec6cuSIpEvPk+vRo8d1PVDvzJkz+u9//6u77rpLU6dOVefOnZWRkaFOnTopKyvLeq2LJG3fvl2FhYXy8fHRhQsXVFhYaPd+8PvAzAgAALjl2HP7XUdavHixZs2apTvuuEMeHh4qLS1Vly5d1KNHD/3lL3/R4MGDFRYWpkaNGllPV7oWX19f7dy5U23atFFxcbGWLVsmb29vnThxoty6L774oh577DHNnDlTkZGRuuOOO6xtI0aMUGxsrFJSUvTnP/9ZISEh1rbw8HDNnTtXffr0kSQ1adJEb7/99nW99/z8fA0ePFjnzp2TxWJRaGioHn74Yfn6+uqTTz7RE088oUmTJqmoqEgBAQFKSUnRH/7wBz300ENq3bq1vLy8tG3btuvaJ5yXxbjexyQ6MX9/fx06dMjsMpSamqqYmBizy4CTYnzAFsYHroaxUXElJSXas2ePmjdvfl3f4P+e5Ofny9fX1+wy4KQqY3xc7e/sWp/POU0LAAAAgCkIIwAAAABMQRgBAAAAYArCCAAAAABTcDctB8i/UKS/r/rR7DKchtl3LAEAAIBzYmYEAAAAgCkIIwAAAJUsKChIt912W5knlLdv316bNm0yr6gr2Ldvn6KiotS2bVstWbLEYfuJi4vTxo0bJUkpKSnasmWLw/blaDf7mP3jH/9QeHi4WrZsqaioKD344IPKyclRdna2atWqdeMFX8O///1vtWnTRs2bN9fdd9+tw4cP39T+CSMAAAAmuHjxohYtWmR2GTZ9/PHH6tChgzIyMvTII4/Yvd2vQ5Y9kpKS1L17d0nOF0ZKS0tVWlpq9/o385hNmzZNycnJ+vzzz/Xzzz/r+++/V1xcnI4dO2Z3vzeitLRUw4YN07x587Rnzx717dtXEyZMuKn7IIwAAACYYPr06Zo5c6bOnz9fri03N1cDBw5URESEwsPDlZiYaG0LCgrSs88+q44dO6pp06aaNWuWte3w4cMaPHiwIiIi1Lp1a02dOrXC/S1dulSvvvqqVq1apcjISP3000/KzMxUz5491bp1a0VGRiolJcXaj8Vi0bRp09ShQwf9/e9/V2xsrEaPHq2ePXuqadOmevTRR7V161Z169ZNwcHBmjhxonXbbt26KSUlRZ9++qnWrFmjuXPnKjIyUklJSbrvvvu0bNky67rr1q0r88T4y44fP67evXtb3/uvg8CcOXMUERGhNm3aKDo62nrM586dq7CwMEVERGjYsGHKz8+3/m4GDRqkmJgYhYeH6+jRo0pNTVXnzp3Vrl073X777daZnF+70WP2a+fOndNLL72kRYsWyd/f37q8R48euv3228vte9iwYWrfvr1at26te++91xpYjh8/rj/+8Y/ljsuWLVvUrl07RUZGKjw8XG+99Va5Pr/77jt5eHhYg2J8fLz+9a9/qaCgoNy6FcUF7AAA4JYT9maYw/re9dguu9Zr06aNunfvrldffVVPP/10mbbHH39cLVq00KpVq5Sbm6t27dpZP0hLUl5entLT0/XLL7+oWbNmGjRokHx9fTV8+HD17t1bH3/8saRLH0Qr0t8jjzyihx56SPv371deXp7mzZsnSbrjjjv06KOPKj4+Xnv37lV0dLTatm2rwMBASZK7u7u+/fZbSVJsbKx+/PFHbdy4UW5ubmrVqpVOnTql9evXq7CwUMHBwRo5cqTCwv7vd9G3b1/1799fkZGR1m/gAwMDNW3aNA0dOlTSpdOWxo0bV+54Jicnq2nTplq3bp0k6eTJk5Kkd999VytXrtTmzZvl6+urU6dOqVq1avrss8+0ePFipaenq1atWho9erSmTJli/VCenp6ujIwM1a9fX/v379f06dOVmpoqHx8fZWZm6q677lJ2draqVatmreFGj9mv7dq1S1WrVlWrVq1sD6T/NW/ePNWtW1eS9OKLL2r69OlasGCBkpOTFRgYqA0bNpQ5Li+88IKeeOIJPfjgg5KkU6dOleszJyfHWqckeXt7y8fHR0eOHFFwcLBddV0LMyMAAAAmmTlzpl577TWdOHGizPK0tDTFx8dLkurVq6eBAwcqLS3N2n75g3mdOnUUHBysAwcO6OzZs9q8ebMmTZpkXe/yh9Pr7S8rK6tcrWfOnNH333+vkSNHSpJCQ0PVuXNnff3119Z1Hn300TLbDBgwQJ6enqpataoiIiIUExOjKlWqqGbNmmrVqpX27t17zWPUq1cv5efnKyMjQwcOHNDWrVt1//33l1svOjpan332mSZNmqTVq1erZs2akqS1a9dqzJgx8vX1lSTVrl1b7u7uSktL05/+9CfrdRdjx47V+vXrrf317dtX9evXlyR9/vnnyszMVJcuXRQZGanBgwfLzc1NOTk5NmuvyDGrqGXLlql9+/YKDw9XUlKStm/fbj0uaWlp5Y5L9+7dNXPmTD333HPavHmzateufVPquF7MjAAAgFuOvbMXjhYUFKShQ4eWOdXqSiwWS5nXnp6e1p/d3d2v+xqNm9Xfb/vx8vKy2W9F9zN+/HjNnz9f9evX16OPPlpmNuKyjh07avv27UpLS9OqVas0depUZWRk2NX/td6LYRjq1atXmdPFKupax+yyVq1aqbCwUD/99NM1Z0c2b96s119/Xenp6apXr57WrFmjZ599VtKl4/L1119r69atZY7LhAkTNGDAAKWlpempp55SeHi43nzzzTL9BgQE6MCBA9bXZ86cUX5+vho1alSRt35FzIwAAACY6JlnnlFycrKOHDliXdazZ0+9/fbbki6darVq1Sr16tXLZj9eXl7q0qWLXnnlFeuyy6dpVaS/3/L29lZUVJT1DlGZmZnavHmzunTpcl39XIuPj4/12o3LRowYodTUVC1ZskRjxoy54nZZWVny8vLS/fffr/nz52vPnj06e/as+vfvrwULFlj7zMvLU0lJiXr27KkVK1bo9OnTkqTExET17t37in3HxMQoLS1NP/zwg3XZ1q1br/lebuSYeXl56YknntCoUaPK3MFq48aN5fZ96tQpeXt7y8/PT4WFhWWuCcrKylLNmjXLHZfdu3eradOmGjVqlJ566qkr3jSgXbt2Kioqsl4fk5iYqH79+pUJlTeKmREAAAAT1alTR+PHj7d+ky1Jr7/+usaOHauIiAgZhqGnn376ihdt/9Z7772nxx9/XGFhYapSpYoGDBigGTNmVLi/33r//fc1ZswYvfHGG7JYLEpKSlJAQMB192PLiBEjFBsbq5SUFP35z39WXFycatSooYEDB+rIkSNq0qTJFbfbtGmTEhISrDMuc+fOla+vr0aMGKEjR46oU6dO8vDwUM2aNZWWlqZ77rlHO3fuVMeOHeXm5qbWrVuXmxm4LCQkRMuWLVN8fLzOnz+vwsJCtW3b1q6Zkhs5Zs8995zq1KmjmJgYlZSUyGKxKDIyUnPmzFFJSYl1vT59+ig5OVktWrSQn5+fevbsaQ0wmzZt0ssvv6wqVaqUOS7PPPOMNmzYoKpVq8rd3b1MiL3Mzc1NycnJio+PV0FBgRo1aqT33nvPrtrtZTEMw7ipPZrI399fhw4dMrsMrUhZq4zSwGuveIvgCexlpaamKiYmxuwy4KQYH7gaxkbFlZSUaM+ePWrevLnc3d3NLsch8vPzrddEuKKSkhK1a9dO8+fP11133WV2Ob87lTE+rvZ3dq3P55ymBQAAAKe1Zs0aNWvWTB07diSIuCBO0wIAAIDT6t+/v/r37292GXAQZkYAAAAAmIIwAgAAAMAUhBEAAAAApiCMAAAAADAFYQQAAACAKQgjAAAAlSwoKEi33XabiouLrcvat2+vTZs2mVfUFezbt09RUVFq27at9SnijhAXF2d9yndKSsoVnwb+e3Gzj9k//vEPhYeHq2XLloqKitKDDz6onJwcZWdnq1atWjde8DUMHjxYjRo1ksViUV5e3k3vn1v7AgAAmODixYtatGiR4uPjzS7lqj7++GN16NBBiYmJ17VdcXGxPDzs/5iZlJRk/TklJUWRkZGKjo6+rn06SmlpqaRLTyO3x808ZtOmTdO6dev0+eefy9/fX5L0xRdf6NixY6pXr9519V9RY8aM0Ztvvqn69es7pH9mRgAAAEwwffp0zZw5U+fPny/Xlpubq4EDByoiIkLh4eFlPtgGBQXp2WefVceOHdW0aVPNmjXL2nb48GENHjxYERERat26taZOnVrh/pYuXapXX31Vq1atUmRkpH766SdlZmaqZ8+eat26tSIjI5WSkmLtx2KxaNq0aerQoYP+/ve/KzY2VqNHj1bPnj3VtGlTPfroo9q6dau6deum4OBgTZw40bptt27dlJKSok8//VRr1qzR3LlzFRkZqaSkJN13331atmyZdd1169bpjjvuKHfMjh8/rt69e1vf+yOPPGJtmzNnjiIiItSmTRtFR0dbj/ncuXMVFhamiIgIDRs2TPn5+dbfzaBBgxQTE6Pw8HAdPXpUqamp6ty5s9q1a6fbb7/dOpPzazd6zH7t3Llzeumll7Ro0SJrEJGkHj166Pbbby+372HDhql9+/Zq3bq17r33Xh07dsx6XP74xz+WOy5btmxRu3btFBkZqfDwcL311lvl+pSknj17OjT4MDMCAABuPWFhjut71y67VmvTpo26d++uV199VU8//XSZtscff1wtWrTQqlWrlJubq3bt2lk/SEtSXl6e0tPT9csvv6hZs2YaNGiQfH19NXz4cPXu3Vsff/yxpEsfRCvS3yOPPKKHHnpI+/fvV15enubNmydJuuOOO/Too48qPj5ee/fuVXR0tNq2bavAwEBJkru7u7799ltJUmxsrH788Udt3LhRbm5uatWqlU6dOqX169ersLBQwcHBGjlypMJ+9bvo27ev+vfvr8jISE2YMEGSFBgYqGnTpmno0KGSLp22NG7cuHLHMzk5WU2bNtW6deskSSdPnpQkvfvuu1q5cqU2b94sX19fnTp1StWqVdNnn32mxYsXKz09XbVq1dLo0aM1ZcoU64fy9PR0ZWRkqH79+tq/f7+mT5+u1NRU+fj4KDMzU3fddZeys7NVrVo1aw03esx+bdeuXapatapatWpleyD9r3nz5qlu3bqSpBdffFHTp0/XggULlJycrMDAQG3YsKHMcXnhhRf0xBNP6MEHH5QknTp1yq793GzMjAAAAJhk5syZeu2113TixIkyy9PS0qynb9WrV08DBw5UWlqatf3yB/M6deooODhYBw4c0NmzZ7V582ZNmjTJut7lD6fX219WVla5Ws+cOaPvv/9eI0eOlCSFhoaqc+fO+vrrr63rPProo2W2GTBggDw9PVW1alVFREQoJiZGVapUUc2aNdWqVSvt3bv3mseoV69eys/PV0ZGhg4cOKCtW7fq/vvvL7dedHS0PvvsM02aNEmrV69WzZo1JUlr167VmDFj5OvrK0mqXbu23N3dlZaWpj/96U/W6y7Gjh2r9evXW/vr27ev9dSkzz//XJmZmerSpYsiIyM1ePBgubm5KScnx2btFTlmFbVs2TK1b99e4eHhSkpK0vbt263HJS0trdxx6d69u2bOnKnnnntOmzdvVu3atW9KHdeLmREAAHDrsXP2wtGCgoI0dOjQMqdaXYnFYinz2tPT0/qzu7t7mQvh7XGz+vttP15eXjb7reh+xo8fr/nz56t+/fp69NFHy8xGXNaxY0dt375daWlpWrVqlaZOnaqMjAy7+r/WezEMQ7169SpzulhFXeuYXdaqVSsVFhbqp59+uubsyObNm/X6668rPT1d9erV05o1a/Tss89KunRcvv76a23durXMcZkwYYIGDBigtLQ0PfXUUwoPD9ebb755w+/vejEzAgAAYKJnnnlGycnJOnLkiHVZz5499fbbb0u6dKrVqlWr1KtXL5v9eHl5qUuXLnrllVesyy6fplWR/n7L29tbUVFR1jtEZWZmavPmzerSpct19XMtPj4+1ms3LhsxYoRSU1O1ZMkSjRkz5orbZWVlycvLS/fff7/mz5+vPXv26OzZs+rfv78WLFhg7TMvL08lJSXq2bOnVqxYodOnT0uSEhMT1bt37yv2HRMTo7S0NP3www/WZVu3br3me7mRY+bl5aUnnnhCo0aN0uHDh63LN27cWG7fp06dkre3t/z8/FRYWFjmmqCsrCzVrFmz3HHZvXu3mjZtqlGjRumpp54y7Q5mzIwAAACYqE6dOho/frz1m2xJev311zV27FhFRETIMAw9/fTTV7xo+7fee+89Pf744woLC1OVKlU0YMAAzZgxo8L9/db777+vMWPG6I033pDFYlFSUpICAgKuux9bRowYodjYWKWkpOjPf/6z4uLiVKNGDQ0cOFBHjhxRkyZNrrjdpk2blJCQYJ1xmTt3rnx9fTVixAgdOXJEnTp1koeHh2rWrKm0tDTdc8892rlzpzp27Cg3Nze1bt36qjMDISEhWrZsmeLj43X+/HkVFhaqbdu2ds2U3Mgxe+6551SnTh3FxMSopKREFotFkZGRmjNnjkpKSqzr9enTR8nJyWrRooX8/PzUs2dPa4DZtGmTXn75ZVWpUqXMcXnmmWe0YcMGVa1aVe7u7mVC7K/de++92rFjhyQpLCxMoaGhN/UW1BbDMIyb1pvJ/P39dejQIbPL0IqUtcooDTS7DKfxwsAIs0twKqmpqYqJiTG7DDgpxgeuhrFRcSUlJdqzZ4+aN28ud3d3s8txiPz8fOs1Ea6opKRE7dq10/z583XXXXeZXc7vTmWMj6v9nV3r8zmnaQEAAMBprVmzRs2aNVPHjh0JIi6I07QAAADgtPr376/+/fubXQYchJkRAAAAAKYgjAAAAJd2+VaqLnSZLOB0Lv99/fbWxdfCaVoAAMClubm5qUqVKjpx4oT8/Pyu+8PS70FpaWmZuysBv+bo8WEYhk6cOKEqVarIze365joIIwAAwOUFBAQoJydHJ0+eNLsUh7hw4YKqV69udhlwUpUxPqpUqVKh2zwTRgAAgMurWrWqQkJCVFpa6pKna6Wlpalnz55mlwEn5ejxYbFYrntG5DLCCAAAuGVU9APT74GrPkMFN4ezjg/X/YsEAAAA4NQIIwAAAABMQRgBAAAAYArCCAAAAABTEEYAAAAAmIIwAgAAAMAUhBEAAAAApiCMAAAAADAFYQQAAACAKQgjAAAAAExBGAEAAABgCsIIAAAAAFMQRgAAAACYgjACAAAAwBSEEQAAAACmIIwAAAAAMAVhBAAAAIApCCMAAAAATEEYAQAAAGAKwggAAAAAUxBGAAAAAJiCMAIAAADAFIQRAAAAAKYgjAAAAAAwhcPDSFBQkFq0aKHIyEhFRkbqww8/lCTt3btXnTp1UvPmzdWhQwft2rXLuo2tNgAAAACuoVJmRj788ENt375d27dv15/+9CdJUnx8vEaPHq09e/Zo8uTJio2Nta5vqw0AAACAazDlNK3c3Fxt27ZNw4cPlyQNGjRIBw8eVGZmps02AAAAAK6jUsLIQw89pIiICI0cOVLHjx/XwYMH1bBhQ3l4eEiSLBaLAgIClJOTY7MNAAAAgOvwcPQOvvrqKwUEBKioqEjPPPOMHn74Yc2cOfOm9J2QkKCEhATr67y8PKWmpt6Uvm9EVZWordsBs8twGqmpR8wuwakUFBQ4xTiFc2J84GoYG7CF8QFbnHl8WAzDMCprZ0ePHlXz5s21b98+hYSE6OTJk/Lw8JBhGGrYsKE2b94sHx+fq7aFhITY7N/f31+HDh2qpHdzdStS1iqjNNDsMpzGCwMjzC7BqaSmpiomJsbsMuCkGB+4GsYGbGF8wBYzx8e1Pp879DStc+fOKS8vz/p6+fLlatu2rerVq6eoqCglJydLklauXCl/f3+FhITYbAMAAADgOhx6mtZ///tfDRo0SCUlJTIMQ8HBwVq6dKkkKTExUbGxsZo9e7Z8fHy0ZMkS63a22gAAAAC4BoeGkeDgYGVkZFyxrUWLFkpPT7/uNgAAAACugSewAwAAADAFYQQAAACAKQgjAAAAAExBGAEAAABgCsIIAAAAAFMQRgAAAACYgjACAAAAwBSEEQAAAACmIIwAAAAAMAVhBAAAAIApCCMAAAAATEEYAQAAAGAKwggAAAAAUxBGAAAAAJiCMAIAAADAFIQRAAAAAKYgjAAAAAAwBWEEAAAAgCkIIwAAAABMQRgBAAAAYArCCAAAAABTEEYAAAAAmIIwAgAAAMAUhBEAAAAApiCMAAAAADAFYQQAAACAKQgjAAAAAExBGAEAAABgCsIIAAAAAFMQRgAAAACYgjACAAAAwBSEEQAAAACmIIwAAAAAMAVhBAAAAIApCCMAAAAATEEYAQAAAGAKwggAAAAAUxBGAAAAAJiCMAIAAADAFIQRAAAAAKYgjAAAAAAwBWEEAAAAgCkIIwAAAABMQRgBAAAAYArCCAAAAABTEEYAAAAAmIIwAgAAAMAUhBEAAAAApiCMAAAAADAFYQQAAACAKQgjAAAAAExBGAEAAABgCsIIAAAAAFMQRgAAAACYgjACAAAAwBSEEQAAAACmIIwAAAAAMAVhBAAAAIApCCMAAAAATEEYAQAAAGAKwggAAAAAUxBGAAAAAJiCMAIAAADAFIQRAAAAAKYgjAAAAAAwBWEEAAAAgCkIIwAAAABMQRgBAAAAYArCCAAAAABTEEYAAAAAmIIwAgAAAMAUhBEAAAAApiCMAAAAADBFpYWRJUuWyGKxKCUlRZKUm5urPn36KDQ0VOHh4frqq6+s69pqAwAAAOAaKiWMZGdn6+2331Z0dLR12ZQpUxQdHa29e/dqyZIlGjp0qIqKiq7ZBgAAAMA1ODyMlJaWKi4uTvPnz1e1atWsy1esWKExY8ZIkjp06KBGjRrpyy+/vGYbAAAAANfg4egdJCQk6M4771S7du2sy06cOKGioiI1aNDAuiwoKEg5OTk2267Ud0JCgvV1Xl6eUlNTHfRO7FdVJWrrdsDsMpxGauoRs0twKgUFBU4xTuGcGB+4GsYGbGF8wBZnHh8ODSM7d+7UypUrHXbNx8SJEzVx4kTra39/f8XExDhkX9djRcpaZZQGml2G03ghJsLsEpxKamqqU4xTOCfGB66GsQFbGB+wxZnHh0NP0/r666+VnZ2t0NBQBQUFacuWLRo9erRWrFghDw8PHTt2zLpudna2AgIC5Ofnd9U2AAAAAK7DoWFk7NixOnr0qLKzs5Wdna3o6GgtXLhQY8eO1ZAhQ7RgwQJJ0rfffqvDhw+ra9eukmSzDQAAAIBrcPg1I1czZ84cjRgxQqGhoapataqSk5NVpUqVa7YBAAAAcA2VGkY2bdpk/bl+/fpat27dFdez1QYAAADANfAEdgAAAACmIIwAAAAAMAVhBAAAAIApCCMAAAAATEEYAQAAAGAKwggAAAAAUxBGAAAAAJiCMAIAAADAFIQRAAAAAKYgjAAAAAAwBWEEAAAAgCkIIwAAAABMQRgBAAAAYArCCAAAAABTEEYAAAAAmIIwAgAAAMAUdoWRgwcP6r777lNkZKQkafv27Xr11VcdWRcAAAAAF2dXGImPj9cDDzwgwzAkSeHh4Vq8eLFDCwMAAADg2uwKI7m5uRo+fLjc3C6t7uHhIQ8PD4cWBgAAAMC12RVGPDw8rLMiknTq1KkyrwEAAADgetkVRoYMGaL4+HidPn1aSUlJ6tWrl+Li4hxdGwAAAAAXZte5VpMmTdLy5cuVn5+vdevWaeLEiRo6dKijawMAAADgwuy+8OPBBx/Ugw8+6MhaAAAAANxC7AojjzzyiCwWS7nl3FELAAAAQEXZFUbat29v/bmgoEArV65UVFSUw4oCAAAA4PrsCiN//vOfy7weO3as+vfv75CCAAAAANwa7Lqb1m95enrq0KFDN7sWAAAAALcQu2ZGJk6caP25pKRE27ZtU3h4uMOKAgAAAOD67Aojvr6+/7eBh4fGjx+vgQMHOqwoAAAAAK7PrjAybdo0R9cBAAAA4BZjM4y8/vrrNjceP378TS0GAAAAwK3DZhjJyMi4atuVnjsCAAAAAPayGUaWLFlSWXUAAAAAuMXYdc2IJB05ckQ7d+5UQUGBdRnPGgEAAABQUXaFkcWLF+u5557TyZMnFRoaqh07dig6OpowAgAAAKDC7Hro4auvvqqMjAw1a9ZM3333nTZs2KDmzZs7ujYAAAAALsyuMFK1alXVrl1bxcXFkqQuXbpo+/btjqwLAAAAgIuz6zStatWqyTAMNW/eXPPmzVNgYKDOnj3r6NoAAAAAuDC7wsisWbN0+vRpvfTSSxozZozy8vL05ptvOro2AAAAAC7MrjDStWtXubu7y9fXV+vXr3d0TQAAAABuAXZdM+Lv768nn3xSP/30k6PrAQAAAHCLsCuMpKenq2bNmrrvvvt0xx13KDExUadPn3Z0bQAAAABcmF1hJCgoSNOnT9f+/fs1Z84cbdy4UQ0bNnR0bQAAAABcmF1h5LLvvvtOH330kTZs2KDu3bs7qiYAAAAAtwC7LmB/5ZVX9M4776ikpESPPPKIduzYwcwIAAAAgBtiVxjZvXu3Fi5cqI4dOzq6HgAAAAC3CLvCyMKFCx1dBwAAAIBbzHVdMwIAAAAANwthBAAAAIApCCMAAAAATGFXGImOjtayZctUVFTk6HoAAAAA3CLsCiPPPfecVqxYoaCgIE2dOlWHDx92dF0AAAAAXJxdYaR3795KSUlRenq6SkpK1KFDBw0ZMkT/8z//4+j6AAAAALio67pm5NSpU/rvf/8rNzc3NWzYUOPGjdO4ceMcVRsAAAAAF2ZXGPnggw905513avjw4YqOjtbevXv1+uuva9u2bfrkk08cXSMAAAAAF2TXQw/ff/99zZgxQz179iyz3N3dXa+//rpDCgMAAADg2uyaGfnjH/9YLogsXrxYktSvX7+bXxUAAAAAl2dXGHnjjTfKLfvHP/5x04sBAAAAcOuweZrW1q1blZ6eruPHj5c5HSs/P18XL150eHEAAAAAXJfNMHL06FFt375d58+fV0ZGhnW5j4+P3nnnHUfXBgAAAMCF2QwjAwYM0IABA/TZZ5/pnnvuqayaAAAAANwCbIaRL7/8Ul27dlVRUZHWrFlTrr1///4OKwwAAACAa7MZRpKTk9W1a1e9+uqr5dosFgthBAAAAECF2Qwjb7/9tiRp48aNlVIMAAAAgFuHXQ89lKSff/5Ze/fuVXFxsXXZwIEDHVIUAAAAANdnVxiZNGmS3n//fbVq1Uru7u6SLp2mRRgBAAAAUFF2hZHVq1dr//79qlGjhqPrAQAAAHCLsOsJ7E2aNJGnp6ejawEAAABwC7FrZuSll17SkCFDFBMTUyaUPPTQQw4rDAAAAIBrsyuMLFiwQD/88IMMwyhzzQhhBAAAAEBF2RVGNm3apN27d8vDw+6bbwEAAACATXZdMxIcHCzDMBxdCwAAAIBbiF1THcHBwerWrZsGDBhQ5pqR8ePHX3Pb3r1769ixY3Jzc5O3t7def/11tW3bVnv37tXDDz+sX375Rb6+vnrnnXcUFhYmSTbbAAAAALgGu8JIYWGhmjdvrp9//tm6zGKx2LWDFStWqFatWpKkf/7zn4qNjdWOHTsUHx+v0aNHKzY2Vh9//LFiY2P17bffSpLNNgAAAACu4ZphpKSkRPXq1dOcOXMqtIPLQUSS8vPzZbFYlJubq23btmndunWSpEGDBmncuHHKzMyUj4/PVdtCQkIqVAMAAAAA53PNMOLu7q6NGzfe0E4eeughax+ffvqpDh48qIYNG1oviLdYLAoICFBOTo58fX2v2kYYAQAAAFyHXadp9e3bV88//7weeeQReXl5WZf7+PjYtZOlS5dKkt59911NnjxZM2fOrECp5SUkJCghIcH6Oi8vT6mpqTel7xtRVSVq63bA7DKcRmrqEbNLcCoFBQVOMU7hnBgfuBrGBmxhfMAWZx4fFsOO22S5uZW/6ZbFYlFJScl177B69erKzs5WaGioTp48KQ8PDxmGoYYNG2rz5s3y8fFRSEjIFduuNTPi7++vQ4cOXXdNN9uKlLXKKA00uwyn8cLACLNLcCqpqamKiYkxuww4KcYHroaxAVsYH7DFzPFxrc/ndt3at7S0tNw/e4JIXl6ejhz5v2/FU1JS5Ofnp3r16ikqKkrJycmSpJUrV8rf318hISE22wAAAAC4DrufYnjw4EF9/fXXkqSuXbuqcePG19wmPz9fQ4YM0YULF+Tm5qa6detq7dq1slgsSkxMVGxsrGbPni0fHx8tWbLEup2tNgAAAACuwa4wsnr1ao0cOVKdO3eWxWLRhAkTtGjRIvXr18/mdoGBgdq6desV21q0aKH09PTrbgMAAADgGuwKIzNmzNCWLVusp0plZmbq/vvvv2YYAQAAAICrseuakZKSkjLXbISEhKi0tNRhRQEAAABwfXaFkXr16ikpKcl68fqiRYtUt25dR9cGAAAAwIXZFUYWLFigpKQkVa9eXdWrV1dSUpISExMdXRsAAAAAF2bzmpF3331XDz/8sIqKirRlyxadPXtWkso8+BAAAAAAKsLmzMjrr78uSRo6dKikSyGEIAIAAADgZrjm3bTGjh2rw4cPa+LEieXaEhISHFIUAAAAANdnM4x89NFH+uijj+Tm5iZfX9/KqgkAAADALcBmGAkODtbkyZPVqFEjjRgxorJqAgAAAHALsOuhhyNGjNDRo0eVlZWl4uJi6/IuXbo4rDAAAAAArs2uMPL8889r7ty5Cg4Olru7uyTJYrFo69atDi0OAAAAgOuyK4wsXrxY+/btk5+fn6PrAQAAAHCLsOuhh/Xr1yeIAAAAALip7JoZ6dWrlyZMmKChQ4fK09PTurx169YOKwwAAACAa7MrjCxdulSStHr1ausyi8Wi/fv3O6YqAAAAAC7PrjCSlZXl6DoAAAAA3GLsCiOStHXrVqWlpUmSevfurfbt2zusKAAAAACuz64L2BcuXKjBgwcrNzdXx48f16BBg5SUlOTo2gAAAAC4MLtmRt544w199913qlu3riTpqaeeUo8ePRQXF+fQ4gAAAAC4LrtmRiRZg8hvfwYAAACAirArjISGhurpp59WTk6OcnJyNHXqVIWGhjq6NgAAAAAuzK4wsmDBAu3bt09RUVFq166dMjMz9dZbbzm6NgAAAAAuzK5rRurWrasPPvjA0bUAAAAAuIXYNTMybdo0nThxwvr6l19+0YwZMxxWFAAAAADXZ1cYWb16tfz8/Kyv69SpU+Zp7AAAAABwvewKI6WlpeWWFRYW3vRiAAAAANw67AojLVq00EsvvaSSkhIVFxdrzpw5uu222xxdGwAAAAAXZlcYee211/T555+revXqqlmzptLS0jR//nxH1wYAAADAhdl1N61GjRppw4YNOnfunCSpZs2aDi0KAAAAgOuzK4xI0tGjR5WVlaXi4mLrsi5dujikKAAAAACuz64w8vzzz2vu3LkKDg6Wu7u7JMlisWjr1q0OLQ4AAACA67IrjCxevFj79u0rc3tfAAAAALgRdl3AXr9+fYIIAAAAgJvKrpmRXr16acKECRo6dKg8PT2ty1u3bu2wwgAAAAC4NrvCyNKlSyWpzFPXLRaL9u/f75iqAAAAALg8u8JIVlaWo+sAAAAAcIux+9a+W7duVVpamiSpd+/eat++vcOKAgAAAOD67LqAfeHChRo8eLByc3N1/PhxDRo0SElJSY6uDQAAAIALs2tm5I033tB3332nunXrSpKeeuop9ejRQ3FxcQ4tDgAAAIDrsmtmRJI1iPz2ZwAAAACoCLvCSGhoqJ5++mnl5OQoJydHU6dOVWhoqKNrAwAAAODC7AojCxYs0L59+xQVFaV27dopMzNTb731lqNrAwAAAODC7LpmpG7duvrggw8cXQsAAACAW4jNMPLVV1/Z3LhLly43tRgAAAAAtw6bYWTSpEnlllksFh05ckRHjx5VSUmJwwoDAAAA4NpshpFvv/22zOuTJ09q1qxZSk5O1owZMxxaGAAAAADXZtcF7AUFBXrhhRfUqlUrSdLPP/+sZ555xqGFAQAAAHBtNsNIaWmpFi5cqNDQUP3nP//Rv//9byUkJMjPz6+y6gMAAADgomyephUeHq6LFy9q9uzZatOmjfLz8/XDDz9Y21u3bu3wAgEAAAC4Jpth5Pz587JYLHr22WfLtVksFu3fv99hhQEAAABwbTbDSHZ2diWVAQAAAOBWY9cF7AAAAABwsxFGAAAAAJiCMAIAAADAFIQRAAAAAKYgjAAAAAAwBWEEAAAAgCkIIwAAAABMQRgBAAAAYArCCAAAAABTEEYAAAAAmIIwAgAAAMAUhBEAAAAApiCMAAAAADCFh9kFwPX9fdWPZpfgNF4YGGF2CQAAAE6DmREAAAAApiCMAAAAADAFYQQAAACAKQgjAAAAAExBGAEAAABgCsIIAAAAAFMQRgAAAACYwqFhpKCgQP/v//0/NW/eXG3atFGvXr2UmZkpScrNzVWfPn0UGhqq8PBwffXVV9btbLUBAAAAcA0OnxkZPXq0du/erR07dmjAgAGKi4uTJE2ZMkXR0dHau3evlixZoqFDh6qoqOiabQAAAABcg0PDiKenp/r27SuLxSJJio6OVnZ2tiRpxYoVGjNmjCSpQ4cOatSokb788strtgEAAABwDZV6zchrr72mAQMG6MSJEyoqKlKDBg2sbUFBQcrJybHZBgAAAMB1eFTWjmbPnq3MzEx98cUXunDhwk3pMyEhQQkJCdbXeXl5Sk1NvSl934iqKlFbtwNmlwEnlJp6RAUFBU4xTuGcGB+4GsYGbGF8wBZnHh+VEkZefvllrVq1SmlpaapRo4Zq1KghDw8PHTt2zDoDkp2drYCAAPn5+V217bcmTpyoiRMnWl/7+/srJiamMt6STStS1iqjNNDsMuCEXoiJUGpqqlOMUzgnxgeuhrEBWxgfsMWZx4fDT9NKSEjQ8uXLtX79etWqVcu6fMiQIVqwYIEk6dtvv9Xhw4fVtWvXa7YBAAAAcA0OnRk5dOiQJk2apODgYHXv3l2SVK1aNf373//WnDlzNGLECIWGhqpq1apKTk5WlSpVJMlmGwAAAADX4NAw4u/vL8MwrthWv359rVu37rrbAAAAALgGnsAOAAAAwBSEEQAAAACmIIwAAAAAMAVhBAAAAIApCCMAAAAATEEYAQAAAGAKwggAAAAAUxBGAAAAAJiCMAIAAADAFIQRAAAAAKYgjAAAAAAwBWEEAAAAgCkIIwAAAABMQRgBAAAAYArCCAAAAABTEEZuAUl7/qikPX80uwwAAADXFxZ26R/sQhgBAAAAYArCCAAAAABTEEYAAAAAmIIwAgAAAMAUhBEAAAAApiCMAAAAADAFYQQAAACAKQgjAAAAAExBGAEAAABgCsIIAAAAAFMQRgAAAACYgjACAAAAwBSEEQAAAACmIIwAAAAAMAVhBAAAAIApCCMAAAAATEEYAQAAAGAKwggAAAAAUxBGAAAAAJiCMAIAAADAFIQRAAAAAKYgjAAAAAAwBWEEAAAAgCkIIwAAAABMQRgBAAAAYArCCAAAAABTEEYAAAAAmIIwAgAAAMAUhBEAAAAApiCMAAAAADAFYQQAAACAKQgjAAAAAExBGAEAAABgCsIIAAAAAFMQRgAAAACYgjACAAAAwBSEEQAAAACmIIwAAAAAMAVhBAAAAIApCCMAAAAATEEYAQAAAGAKwggAAAAAUxBGAAAAAJiCMAIAAADAFIQRAAAAAKYgjAAAAAAwBWEEAAAAgCkIIwAAAABMQRgBAAAAYArCCAAAAABTEEYAAAAAmIIwAgAAAMAUhBEAAAAApiCMAAAAADAFYQQAAACAKQgjAAAAAEzh8DAyfvx4BQUFyWKxaPv27dble/fuVadOndS8eXN16NBBu3btsqsNAAAAgGtweBgZPHiwNm/erMDAwDLL4+PjNXr0aO3Zs0eTJ09WbGysXW0AAAAAXIPDw0iXLl3k7+9fZllubq62bdum4cOHS5IGDRqkgwcPKjMz02YbAAAAANdhyjUjBw8eVMOGDeXh4SFJslgsCggIUE5Ojs02AAAAAK7Dw+wCbkRCQoISEhKsr/Py8pSammpiRZdUVYnauh0wuwwrTxVKklPVdKtKTT2igoICpxincE6MD1wNYwO2MD6cx51nz0qS/seJfh/OPD5MCSNNmjTR0aNHVVxcLA8PDxmGoZycHAUEBMjHx+eqbb81ceJETZw40fra399fMTExlflWrmhFylpllAZee8VKUqCqkuRUNd2qXoiJUGpqqlOMUzgnxgeuhrEBWxgfTsTLS5Kc6vfhzOPDlNO06tWrp6ioKCUnJ0uSVq5cKX9/f4WEhNhsAwAAAOA6HD4zEh8fr08++UTHjh1TTEyMvL29lZmZqcTERMXGxmr27Nny8fHRkiVLrNvYagMAAADgGhweRhITE6+4vEWLFkpPT7/uNgAAAACugSewAwAAADAFYQQAAACAKQgjAAAAAExBGAEAAABgCsIIAAAAAFMQRgAAAACYgjACAAAAwBSEEQAAAACmIIwAAAAAMAVhBAAAAIApCCMAAAAATEEYAQAAAGAKwggAAAAAUxBGAAAAAJiCMAIAAADAFIQRAAAAAKYgjAAAAAAwBWEEAAAAgCkIIwAAAABMQRgBAAAAYArCCAAAAABTEEYAAAAAmIIwAgAAAMAUhBEAAAAApiCMAAAAADAFYQQAAACAKQgjAAAAAExBGAEAAABgCsIIAAAAAFMQRgAAAACYgjACAAAAwBSEEQAAAACm8DC7AOBW8vdVP6qtW5H+vupHs0txCi8MjDC7BAAAYCJmRgAAAACYgjACAAAAwBSEEQAAAACmIIwAAAAAMAVhBAAAAIApCCMAAAAATEEYAQAAAGAKwggAAAAAUxBGAAAAAJiCMAIAAADAFIQRAAAAAKYgjAAAAAAwBWEEAAAAgCkIIwAAAABMQRgBAAAAYArCCAAAAABTEEYAAAAAmIIwAgAAAMAUhBEAAAAApiCMAAAAADCFh9kFAAAAoGL+vupHSVJbtyLrz7eyFwZGmF0CrhMzIwAAAABMwcwIAAAAXIIzzA5NOH1RkjTPCWr5PcwUEUYAAMDvijN84ARwc3CaFgAAAABTMDMCwDR8u1nW72E6HQCAm4mZEQAAAACmYGYEAAAn9/dVP3LrVgAuiZkRAAAAAKZgZgQAnATffv8frp8BgFsDMyMAAAAATEEYAQAAAGAKTtMCADgdTlUDgFsDMyMAAAAATEEYAQAAAGAKwggAAAAAUxBGAAAAAJiCMAIAAADAFIQRAAAAAKZw2jCyd+9ederUSc2bN1eHDh20a9cus0sCAAAAcBM5bRiJj4/X6NGjtWfPHk2ePFmxsbFmlwQAAADgJnLKMJKbm6tt27Zp+PDhkqRBgwbp4MGDyszMNLkyAAAAADeLU4aRgwcPqmHDhvLwuPSAeIvFooCAAOXk5JhcGQAAAICbxcPsAm5EQkKCEhISrK+PHTsmf39/Eyu65OzZs/Ly8jK7DKtq//u/7+keU+vAJW852fiAc2F84GoYG7CF8eE83rv8w3jzP3e9N/7S/5r52fT48eM22y2GYRiVVIvdcnNzFRISopMnT8rDw0OGYahhw4bavHmzQkJCzC7vmvz9/XXo0CGzy4CTYnzAFsYHroaxAVsYH7DFmceHU56mVa9ePUVFRSk5OVmStHLlSvn7+/8ugggAAAAA+zjtaVqJiYmKjY3V7Nmz5ePjoyVLlphdEgAAAICbyGnDSIsWLZSenm52GRUyceJEs0uAE2N8wBbGB66GsQFbGB+wxZnHh1NeMwIAAADA9TnlNSMAAAAAXB9hBAAAAIApCCMVtHfvXnXq1EnNmzdXhw4dtGvXriuut2jRIoWGhqpZs2YaNWqUioqKKrlSmMGe8bFhwwbdfvvtatWqlcLCwvS3v/1NpaWlJlSLymbvfz8kyTAM3X333apVq1blFQjT2Ds2fvzxR3Xr1k0tW7ZUy5YttWrVqkquFGawZ3yUlpZq4sSJatWqlVq3bq3u3bsrMzPThGpRmcaPH6+goCBZLBZt3779qus55edSAxXSvXt3Y8mSJYZhGMZHH31ktG/fvtw6+/fvNxo2bGgcPXrUKC0tNfr162e88cYblVwpzGDP+Pj++++Nffv2GYZhGBcuXDDuvPNO6zZwbfaMj8teeeUVIy4uzvD19a2c4mAqe8bGuXPnjKZNmxpff/21YRiGUVxcbOTm5lZmmTCJPePjn//8p3H77bcbhYWFhmEYxsyZM40hQ4ZUZpkwwZdffmkcPHjQCAwMNDIyMq64jrN+LmVmpAJyc3O1bds2DR8+XJI0aNAgHTx4sNw3Dx9//LH69++vBg0ayGKxaMyYMVq+fLkZJaMS2Ts+2rZtq+DgYEmSp6enIiMjlZ2dXdnlopLZOz4kadeuXUpJSdGUKVMqu0yYwN6xsWzZMkVHR6tz586SJHd3d9WtW7fS60Xlsnd8WCwWXbx4UQUFBTIMQ6dPn5a/v78ZJaMSdenS5Zq/Z2f9XEoYqYCDBw+qYcOG8vC4dGdki8WigIAA5eTklFkvJydHgYGB1tdBQUHl1oHrsXd8/NqxY8f08ccf67777qusMmESe8dHUVGRRo0apcTERLm7u5tRKiqZvWPjp59+UrVq1XTfffcpMjJSDz30kI4fP25GyahE9o6Pfv36qVu3bmrQoIEaNmyoL774Qs8995wZJcPJOOvnUsIIYLLTp0+rX79++tvf/qb27dubXQ6cxIwZMzRw4EC1bNnS7FLgZIqLi5WWlqbExERlZGSocePGGjt2rNllwUls27ZNO3fu1OHDh3XkyBH16NFDY8aMMbss4KoIIxXQpEkTHT16VMXFxZIuXWCak5OjgICAMusFBATowIED1tfZ2dnl1oHrsXd8SNKZM2fUp08fDRgwwKkfSISbx97x8eWXX2r+/PkKCgpS586ddfr0aQUFBfENuAu7nv9v6d69uxo3biyLxaLhw4dry5YtZpSMSmTv+Fi6dKn1phdubm56+OGHtXHjRjNKhpNx1s+lhJEKqFevnqKiopScnCxJWrlypfz9/RUSElJmvUGDBmnNmjU6duyYDMPQggUL9MADD5hRMiqRvePj7Nmz6tOnj/r06aNnnnnGjFJhAnvHx9dff60DBw4oOztbmzdvlo+Pj7Kzs7k2wIXZOzbuv/9+ffvttzp9+rQk6dNPP1WbNm0qvV5ULnvHR3BwsDZs2KDCwkJJ0tq1axUeHl7p9cL5OO3nUvOunf99+89//mNER0cboaGhRrt27YwffvjBMAzDGDlypLF69WrregsXLjSCg4ON4OBg49FHH7Xe3QKuzZ7xMWvWLMPDw8No06aN9d+sWbPMLBuVxN7/flyWlZXF3bRuEfaOjaVLlxphYWFGRESE0adPHyMnJ8esklGJ7BkfBQUFRlxcnHHbbbcZERERRq9evax3boTrGj16tNG4cWPD3d3dqFevntGsWTPDMH4fn0sthmEYZgciAAAAALceTtMCAAAAYArCCAAAAABTEEYAAAAAmIIwAgAAAMAUhBEAAAAApiCMAAAAADAFYQQAAACAKQgjAAAAAExBGAEAAABgiv8PQ1SkWPncxMIAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Function computes prediction sets - here we essentially collect all the steps done above into one function\n",
        "# Function takes training set, test points, underlying classifier, significance level\n",
        "# Function returns for each test point, prediction sets and two sets of p-values for each potential label 0 & 1\n",
        "\n",
        "def prediction_sets(X_train, y_train, X_calib, y_calib, X_test, y_test, model = LogisticRegression(), significance_level = 0.05):\n",
        "\n",
        "  prediction_sets = []\n",
        "  p_values_class_0 = []\n",
        "  p_values_class_1 = []\n",
        "\n",
        "  model.fit(X_train, y_train)\n",
        "\n",
        "  # make point predictions and obtain class scores on the calibration and test sets\n",
        "  y_pred_calib = model.predict(X_calib)\n",
        "  y_pred_score_calib = model.predict_proba(X_calib)\n",
        "\n",
        "  y_pred_test = model.predict(X_test)\n",
        "  y_pred_score_test = model.predict_proba(X_test)\n",
        "\n",
        "  for i,y in enumerate(y_test):\n",
        "        # compute nonconformity scores for each test trying both potential labels - 0 & 1\n",
        "        non_conformity_scores_y_test_class_0 = 1 - y_pred_score_test[i,0]\n",
        "        non_conformity_scores_y_test_class_1 = 1 - y_pred_score_test[i,1]\n",
        "\n",
        "        # compute p-value of the test point twice, each time assigning potential label 0 and 1\n",
        "        p_value_class_0 = compute_p_values(non_conformity_scores_calib_plus_test_0, non_conformity_scores_y_test_class_0)\n",
        "        p_value_class_1 = compute_p_values(non_conformity_scores_calib_plus_test_0, non_conformity_scores_y_test_class_1)\n",
        "\n",
        "        p_values_class_0.append( p_value_class_0)\n",
        "        p_values_class_1.append( p_value_class_1)\n",
        "\n",
        "        if (p_value_class_0 > significance_level) and (p_value_class_1 > significance_level):\n",
        "          prediction_sets.append([True,True])\n",
        "        elif (p_value_class_0 > significance_level) and (p_value_class_1 <= significance_level):\n",
        "          prediction_sets.append([True,False])\n",
        "        elif (p_value_class_0 <= significance_level) and (p_value_class_1 >significance_level):\n",
        "          prediction_sets.append([False,True])\n",
        "        else:\n",
        "          prediction_sets.append([False,False])\n",
        "\n",
        "  return prediction_sets, p_values_class_0, p_values_class_1"
      ],
      "metadata": {
        "id": "s9a15kIErd9K"
      },
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "predictions_test_sets, p_values_test_set_class_0, p_values_test_set_class_1 = prediction_sets(X_train, y_train, X_calib, y_calib, X_test, y_test, model = LogisticRegression(), significance_level = 0.05)"
      ],
      "metadata": {
        "id": "MKOC3OnLrdw0"
      },
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "predictions_test_sets[:20]"
      ],
      "metadata": {
        "id": "HEu_6qv5rdhS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a2322e90-dfb8-4c96-cd07-c7f1e2670ffd"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[[True, False],\n",
              " [True, False],\n",
              " [True, False],\n",
              " [True, True],\n",
              " [True, False],\n",
              " [True, True],\n",
              " [True, False],\n",
              " [True, False],\n",
              " [True, False],\n",
              " [True, False],\n",
              " [True, False],\n",
              " [True, False],\n",
              " [True, True],\n",
              " [True, False],\n",
              " [True, False],\n",
              " [True, True],\n",
              " [True, False],\n",
              " [True, False],\n",
              " [True, False],\n",
              " [True, False]]"
            ]
          },
          "metadata": {},
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's build Inductive Conformal Predictor (ICP) using open source library nonconformist https://github.com/donlnz/nonconformist\n",
        "\n",
        "#nc = NcFactory.create_nc(model,err_func=InverseProbabilityErrFunc())\t# Create a default nonconformity function, this is the inverse probability (hinge loss) nonconformity measure we used above\n",
        "\n",
        "icp = IcpClassifier(ClassifierNc(ClassifierAdapter(LogisticRegression()),\n",
        "                                 MarginErrFunc()))"
      ],
      "metadata": {
        "id": "s-Ljf11wO5e8"
      },
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit the ICP using the proper training set\n",
        "icp.fit(X_train, y_train)\n",
        "\n",
        "# Calibrate the ICP using the calibraiton set\n",
        "icp.calibrate(X_calib, y_calib)"
      ],
      "metadata": {
        "id": "mkXq7DDcQbk-"
      },
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "icp"
      ],
      "metadata": {
        "id": "NC5LEWbrGVSE",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 194
        },
        "outputId": "09a5ce29-0b53-4085-ebee-3659141d8d98"
      },
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "IcpClassifier(condition=<function BaseIcp.__init__.<locals>.<lambda> at 0x7fcfc3206320>,\n",
              "              nc_function=ClassifierNc(err_func=<nonconformist.nc.MarginErrFunc object at 0x7fcfc2e4b160>,\n",
              "                                       model=ClassifierAdapter(fit_params={},\n",
              "                                                               model=LogisticRegression())))"
            ],
            "text/html": [
              "<style>#sk-container-id-2 {color: black;background-color: white;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>IcpClassifier(condition=&lt;function BaseIcp.__init__.&lt;locals&gt;.&lt;lambda&gt; at 0x7fcfc3206320&gt;,\n",
              "              nc_function=ClassifierNc(err_func=&lt;nonconformist.nc.MarginErrFunc object at 0x7fcfc2e4b160&gt;,\n",
              "                                       model=ClassifierAdapter(fit_params={},\n",
              "                                                               model=LogisticRegression())))</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">IcpClassifier</label><div class=\"sk-toggleable__content\"><pre>IcpClassifier(condition=&lt;function BaseIcp.__init__.&lt;locals&gt;.&lt;lambda&gt; at 0x7fcfc3206320&gt;,\n",
              "              nc_function=ClassifierNc(err_func=&lt;nonconformist.nc.MarginErrFunc object at 0x7fcfc2e4b160&gt;,\n",
              "                                       model=ClassifierAdapter(fit_params={},\n",
              "                                                               model=LogisticRegression())))</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">nc_function: ClassifierNc</label><div class=\"sk-toggleable__content\"><pre>ClassifierNc(err_func=&lt;nonconformist.nc.MarginErrFunc object at 0x7fcfc2e4b160&gt;,\n",
              "             model=ClassifierAdapter(fit_params={}, model=LogisticRegression()))</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" ><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">model: ClassifierAdapter</label><div class=\"sk-toggleable__content\"><pre>ClassifierAdapter(fit_params={}, model=LogisticRegression())</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" ><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">model: LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression()</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-6\" type=\"checkbox\" ><label for=\"sk-estimator-id-6\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression()</pre></div></div></div></div></div></div></div></div></div></div></div></div></div></div></div></div></div></div></div></div>"
            ]
          },
          "metadata": {},
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Produce predictions for the test set, with confidence 95%\n",
        "prediction = icp.predict(X_test.values, significance=0.05)"
      ],
      "metadata": {
        "id": "kIDFfnBtGAeO"
      },
      "execution_count": 52,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "prediction.shape"
      ],
      "metadata": {
        "id": "ITwU9s8YsmVK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b12a4326-8077-42b8-eadb-dcb229cadeb3"
      },
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1000, 2)"
            ]
          },
          "metadata": {},
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "prediction[:20:]"
      ],
      "metadata": {
        "id": "_2eE8zrNHzBR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8fa05498-c12b-4dae-ff95-49093df91144"
      },
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[ True, False],\n",
              "       [ True, False],\n",
              "       [ True, False],\n",
              "       [ True,  True],\n",
              "       [ True, False],\n",
              "       [ True,  True],\n",
              "       [ True, False],\n",
              "       [ True, False],\n",
              "       [ True, False],\n",
              "       [ True, False],\n",
              "       [ True, False],\n",
              "       [ True, False],\n",
              "       [ True,  True],\n",
              "       [ True, False],\n",
              "       [ True, False],\n",
              "       [ True,  True],\n",
              "       [ True, False],\n",
              "       [ True, False],\n",
              "       [ True, False],\n",
              "       [ True, False]])"
            ]
          },
          "metadata": {},
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y_test"
      ],
      "metadata": {
        "id": "dTegqH0p04q8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f5a5bf09-5355-4cee-9bda-64e076116040"
      },
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "34262    0\n",
              "25171    0\n",
              "11426    0\n",
              "22639    1\n",
              "34917    0\n",
              "        ..\n",
              "7791     0\n",
              "30960    0\n",
              "23800    0\n",
              "29208    0\n",
              "9604     0\n",
              "Name: class, Length: 1000, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Here we output prediction sets we constructed manually by writing code from scratch and comparing it with the output from Nonconformist. The results are the same.\n",
        "for i in range(len(predictions_test_sets)):\n",
        "  print(predictions_test_sets[i], prediction[i, :])"
      ],
      "metadata": {
        "id": "Lk6kdfoA5If2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1521f95e-4b96-488e-cc40-498ab387e2b4"
      },
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[False, True] [False  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[False, True] [False  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[False, True] [False  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[False, True] [False  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, False] [ True False]\n",
            "[True, True] [ True  True]\n",
            "[True, True] [ True  True]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "assert (predictions_test_sets, prediction)"
      ],
      "metadata": {
        "id": "L-aF9f8p5OUT"
      },
      "execution_count": 57,
      "outputs": []
    }
  ]
}