Concept Hierarchy Generation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "data = pd.read_csv(\"train.csv\")\n",
    "data['Age'] = data['Age'].fillna(data['Age'].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>886</th>\n",
       "      <td>887</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>Montvila, Rev. Juozas</td>\n",
       "      <td>male</td>\n",
       "      <td>27.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>211536</td>\n",
       "      <td>13.0000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>887</th>\n",
       "      <td>888</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Graham, Miss. Margaret Edith</td>\n",
       "      <td>female</td>\n",
       "      <td>19.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>112053</td>\n",
       "      <td>30.0000</td>\n",
       "      <td>B42</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>888</th>\n",
       "      <td>889</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Johnston, Miss. Catherine Helen \"Carrie\"</td>\n",
       "      <td>female</td>\n",
       "      <td>29.699118</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>W./C. 6607</td>\n",
       "      <td>23.4500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>889</th>\n",
       "      <td>890</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Behr, Mr. Karl Howell</td>\n",
       "      <td>male</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>111369</td>\n",
       "      <td>30.0000</td>\n",
       "      <td>C148</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>890</th>\n",
       "      <td>891</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Dooley, Mr. Patrick</td>\n",
       "      <td>male</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>370376</td>\n",
       "      <td>7.7500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>891 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     PassengerId  Survived  Pclass  \\\n",
       "0              1         0       3   \n",
       "1              2         1       1   \n",
       "2              3         1       3   \n",
       "3              4         1       1   \n",
       "4              5         0       3   \n",
       "..           ...       ...     ...   \n",
       "886          887         0       2   \n",
       "887          888         1       1   \n",
       "888          889         0       3   \n",
       "889          890         1       1   \n",
       "890          891         0       3   \n",
       "\n",
       "                                                  Name     Sex        Age  \\\n",
       "0                              Braund, Mr. Owen Harris    male  22.000000   \n",
       "1    Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.000000   \n",
       "2                               Heikkinen, Miss. Laina  female  26.000000   \n",
       "3         Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.000000   \n",
       "4                             Allen, Mr. William Henry    male  35.000000   \n",
       "..                                                 ...     ...        ...   \n",
       "886                              Montvila, Rev. Juozas    male  27.000000   \n",
       "887                       Graham, Miss. Margaret Edith  female  19.000000   \n",
       "888           Johnston, Miss. Catherine Helen \"Carrie\"  female  29.699118   \n",
       "889                              Behr, Mr. Karl Howell    male  26.000000   \n",
       "890                                Dooley, Mr. Patrick    male  32.000000   \n",
       "\n",
       "     SibSp  Parch            Ticket     Fare Cabin Embarked  \n",
       "0        1      0         A/5 21171   7.2500   NaN        S  \n",
       "1        1      0          PC 17599  71.2833   C85        C  \n",
       "2        0      0  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3        1      0            113803  53.1000  C123        S  \n",
       "4        0      0            373450   8.0500   NaN        S  \n",
       "..     ...    ...               ...      ...   ...      ...  \n",
       "886      0      0            211536  13.0000   NaN        S  \n",
       "887      0      0            112053  30.0000   B42        S  \n",
       "888      1      2        W./C. 6607  23.4500   NaN        S  \n",
       "889      0      0            111369  30.0000  C148        C  \n",
       "890      0      0            370376   7.7500   NaN        Q  \n",
       "\n",
       "[891 rows x 12 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "      <td>891.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>446.000000</td>\n",
       "      <td>0.383838</td>\n",
       "      <td>2.308642</td>\n",
       "      <td>29.699118</td>\n",
       "      <td>0.523008</td>\n",
       "      <td>0.381594</td>\n",
       "      <td>32.204208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>257.353842</td>\n",
       "      <td>0.486592</td>\n",
       "      <td>0.836071</td>\n",
       "      <td>13.002015</td>\n",
       "      <td>1.102743</td>\n",
       "      <td>0.806057</td>\n",
       "      <td>49.693429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.420000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>223.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.910400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>446.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>29.699118</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>14.454200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>668.500000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>35.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>31.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>891.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>512.329200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       PassengerId    Survived      Pclass         Age       SibSp  \\\n",
       "count   891.000000  891.000000  891.000000  891.000000  891.000000   \n",
       "mean    446.000000    0.383838    2.308642   29.699118    0.523008   \n",
       "std     257.353842    0.486592    0.836071   13.002015    1.102743   \n",
       "min       1.000000    0.000000    1.000000    0.420000    0.000000   \n",
       "25%     223.500000    0.000000    2.000000   22.000000    0.000000   \n",
       "50%     446.000000    0.000000    3.000000   29.699118    0.000000   \n",
       "75%     668.500000    1.000000    3.000000   35.000000    1.000000   \n",
       "max     891.000000    1.000000    3.000000   80.000000    8.000000   \n",
       "\n",
       "            Parch        Fare  \n",
       "count  891.000000  891.000000  \n",
       "mean     0.381594   32.204208  \n",
       "std      0.806057   49.693429  \n",
       "min      0.000000    0.000000  \n",
       "25%      0.000000    7.910400  \n",
       "50%      0.000000   14.454200  \n",
       "75%      0.000000   31.000000  \n",
       "max      6.000000  512.329200  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part I: \n",
    "    Automatic generation of a concept hierarchy for nomial data based on number of distinct values\n",
    "    of attributes in the given schema"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that: Name, Sex, Ticket, Cabin, Embarked are all nomial attributes. Let us build an automatic generation of \n",
    "concept hierarchy for these attributes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "nomial_data = data[['Name', 'Sex', 'Ticket','Cabin','Embarked']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "list=[]\n",
    "for na in nomial_data:\n",
    " list.append([na,len(set(data[na]))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['Name', 891], ['Sex', 2], ['Ticket', 681], ['Cabin', 148], ['Embarked', 4]]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "list.sort(key=lambda x: x[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['Sex', 2], ['Embarked', 4], ['Cabin', 148], ['Ticket', 681], ['Name', 891]]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we rank hierarchy of these nomial attributes from high to low"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sex\n",
      "Embarked\n",
      "Cabin\n",
      "Ticket\n",
      "Name\n"
     ]
    }
   ],
   "source": [
    "for l in list:\n",
    "    print(l[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part II: \n",
    "    Automatic generation of a concept hierarchy for numeric data based on equal-width partitioning "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us use attribute age as an example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "concept_hierarchy=[]\n",
    "range_min=0\n",
    "range_max=80\n",
    "step=10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Initialize concept_hierarchy array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "i=0\n",
    "j=0\n",
    "while (i<range_max):\n",
    "    name='Level_'+str(j)\n",
    "    min=i\n",
    "    max=i+step\n",
    "    concept_hierarchy.append([name,min,max]) \n",
    "    i+=step\n",
    "    j+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['Level_0', 0, 10],\n",
       " ['Level_1', 10, 20],\n",
       " ['Level_2', 20, 30],\n",
       " ['Level_3', 30, 40],\n",
       " ['Level_4', 40, 50],\n",
       " ['Level_5', 50, 60],\n",
       " ['Level_6', 60, 70],\n",
       " ['Level_7', 70, 80]]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_hierarchy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "for c_hierarchy in concept_hierarchy:\n",
    "    count=0\n",
    "    sum=0\n",
    "    for age in data['Age']:\n",
    "        if age<=c_hierarchy[2] and age>c_hierarchy[1]:\n",
    "            count+=1\n",
    "            sum+=age\n",
    "    level_mean=sum/count\n",
    "    c_hierarchy.append(count)\n",
    "    c_hierarchy.append(sum)\n",
    "    c_hierarchy.append(level_mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['Level_0', 0, 10, 64, 273.16999999999996, 4.268281249999999],\n",
       " ['Level_1', 10, 20, 115, 1991.5, 17.317391304347826],\n",
       " ['Level_2', 20, 30, 407, 11104.24382352941, 27.28315435756612],\n",
       " ['Level_3', 30, 40, 155, 5433.0, 35.05161290322581],\n",
       " ['Level_4', 40, 50, 86, 3902.0, 45.372093023255815],\n",
       " ['Level_5', 50, 60, 42, 2305.5, 54.892857142857146],\n",
       " ['Level_6', 60, 70, 17, 1086.0, 63.88235294117647],\n",
       " ['Level_7', 70, 80, 5, 366.5, 73.3]]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_hierarchy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that: we divide Age attribute into 8 concept hierarchies, width=10, for each tuple, we have \n",
    "level, bin_min, bin_max, count, bin_sum, bin_mean"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part III: \n",
    "    Automatic generation of a concept hierarchy for numeric data based on equal-frequency partitioning "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We still use attribute age as an example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "concept_hierarchy=[]\n",
    "range_min=0\n",
    "range_max=80\n",
    "bin_depth=6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "891"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data['Age'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "Age = data['Age'].sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "80.0"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Age.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "start=0\n",
    "for i in range(int(len(Age)/bin_depth)):\n",
    "    sum=0\n",
    "    name='Level_'+str(i)\n",
    "    min=Age[Age.index[start]]\n",
    "    max=Age[Age.index[start+bin_depth-1]]\n",
    "    for j in range(bin_depth):\n",
    "        sum+=Age[Age.index[start+j]]\n",
    "    mean=sum/bin_depth\n",
    "    concept_hierarchy.append([name,min,max,sum,mean]) \n",
    "    start+=bin_depth\n",
    "    \n",
    "sum=0\n",
    "name='Level_'+str(i+1)\n",
    "if len(Age)>int(len(Age)/bin_depth)*bin_depth:\n",
    "    min=Age[Age.index[start]]\n",
    "    count=len(Age)-int(int(len(Age)/bin_depth))*bin_depth\n",
    "    for j in range(count):\n",
    "        sum+=Age[Age.index[start+j]]\n",
    "    max=Age[Age.index[start+count-1]]\n",
    "    mean=sum/count\n",
    "    concept_hierarchy.append([name,min,max,sum,mean]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['Level_0', 0.42, 0.83, 4.25, 0.7083333333333334],\n",
       " ['Level_1', 0.92, 1.0, 5.92, 0.9866666666666667],\n",
       " ['Level_2', 1.0, 2.0, 10.0, 1.6666666666666667],\n",
       " ['Level_3', 2.0, 2.0, 12.0, 2.0],\n",
       " ['Level_4', 3.0, 3.0, 18.0, 3.0],\n",
       " ['Level_5', 4.0, 4.0, 24.0, 4.0],\n",
       " ['Level_6', 4.0, 5.0, 26.0, 4.333333333333333],\n",
       " ['Level_7', 5.0, 7.0, 35.0, 5.833333333333333],\n",
       " ['Level_8', 7.0, 8.0, 46.0, 7.666666666666667],\n",
       " ['Level_9', 9.0, 9.0, 54.0, 9.0],\n",
       " ['Level_10', 9.0, 11.0, 60.0, 10.0],\n",
       " ['Level_11', 11.0, 14.0, 74.0, 12.333333333333334],\n",
       " ['Level_12', 14.0, 14.5, 84.5, 14.083333333333334],\n",
       " ['Level_13', 15.0, 16.0, 91.0, 15.166666666666666],\n",
       " ['Level_14', 16.0, 16.0, 96.0, 16.0],\n",
       " ['Level_15', 16.0, 16.0, 96.0, 16.0],\n",
       " ['Level_16', 16.0, 17.0, 98.0, 16.333333333333332],\n",
       " ['Level_17', 17.0, 17.0, 102.0, 17.0],\n",
       " ['Level_18', 17.0, 18.0, 103.0, 17.166666666666668],\n",
       " ['Level_19', 18.0, 18.0, 108.0, 18.0],\n",
       " ['Level_20', 18.0, 18.0, 108.0, 18.0],\n",
       " ['Level_21', 18.0, 18.0, 108.0, 18.0],\n",
       " ['Level_22', 18.0, 18.0, 108.0, 18.0],\n",
       " ['Level_23', 18.0, 19.0, 113.0, 18.833333333333332],\n",
       " ['Level_24', 19.0, 19.0, 114.0, 19.0],\n",
       " ['Level_25', 19.0, 19.0, 114.0, 19.0],\n",
       " ['Level_26', 19.0, 19.0, 114.0, 19.0],\n",
       " ['Level_27', 19.0, 20.0, 118.0, 19.666666666666668],\n",
       " ['Level_28', 20.0, 20.0, 120.0, 20.0],\n",
       " ['Level_29', 20.0, 20.5, 120.5, 20.083333333333332],\n",
       " ['Level_30', 21.0, 21.0, 126.0, 21.0],\n",
       " ['Level_31', 21.0, 21.0, 126.0, 21.0],\n",
       " ['Level_32', 21.0, 21.0, 126.0, 21.0],\n",
       " ['Level_33', 21.0, 21.0, 126.0, 21.0],\n",
       " ['Level_34', 22.0, 22.0, 132.0, 22.0],\n",
       " ['Level_35', 22.0, 22.0, 132.0, 22.0],\n",
       " ['Level_36', 22.0, 22.0, 132.0, 22.0],\n",
       " ['Level_37', 22.0, 22.0, 132.0, 22.0],\n",
       " ['Level_38', 22.0, 23.0, 135.0, 22.5],\n",
       " ['Level_39', 23.0, 23.0, 138.0, 23.0],\n",
       " ['Level_40', 23.0, 23.0, 138.0, 23.0],\n",
       " ['Level_41', 23.5, 24.0, 143.5, 23.916666666666668],\n",
       " ['Level_42', 24.0, 24.0, 144.0, 24.0],\n",
       " ['Level_43', 24.0, 24.0, 144.0, 24.0],\n",
       " ['Level_44', 24.0, 24.0, 144.0, 24.0],\n",
       " ['Level_45', 24.0, 24.0, 144.0, 24.0],\n",
       " ['Level_46', 24.0, 25.0, 148.5, 24.75],\n",
       " ['Level_47', 25.0, 25.0, 150.0, 25.0],\n",
       " ['Level_48', 25.0, 25.0, 150.0, 25.0],\n",
       " ['Level_49', 25.0, 25.0, 150.0, 25.0],\n",
       " ['Level_50', 25.0, 26.0, 155.0, 25.833333333333332],\n",
       " ['Level_51', 26.0, 26.0, 156.0, 26.0],\n",
       " ['Level_52', 26.0, 26.0, 156.0, 26.0],\n",
       " ['Level_53', 26.0, 27.0, 161.0, 26.833333333333332],\n",
       " ['Level_54', 27.0, 27.0, 162.0, 27.0],\n",
       " ['Level_55', 27.0, 27.0, 162.0, 27.0],\n",
       " ['Level_56', 27.0, 28.0, 167.0, 27.833333333333332],\n",
       " ['Level_57', 28.0, 28.0, 168.0, 28.0],\n",
       " ['Level_58', 28.0, 28.0, 168.0, 28.0],\n",
       " ['Level_59', 28.0, 28.0, 168.0, 28.0],\n",
       " ['Level_60', 28.0, 29.0, 171.0, 28.5],\n",
       " ['Level_61', 29.0, 29.0, 174.0, 29.0],\n",
       " ['Level_62', 29.0, 29.0, 174.0, 29.0],\n",
       " ['Level_63', 29.0, 29.0, 174.0, 29.0],\n",
       " ['Level_64',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_65',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_66',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_67',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_68',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_69',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_70',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_71',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_72',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_73',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_74',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_75',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_76',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_77',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_78',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_79',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_80',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_81',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_82',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_83',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_84',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_85',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_86',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_87',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_88',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_89',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_90',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_91',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_92',\n",
       "  29.69911764705882,\n",
       "  29.69911764705882,\n",
       "  178.1947058823529,\n",
       "  29.699117647058817],\n",
       " ['Level_93', 29.69911764705882, 30.0, 179.09735294117647, 29.84955882352941],\n",
       " ['Level_94', 30.0, 30.0, 180.0, 30.0],\n",
       " ['Level_95', 30.0, 30.0, 180.0, 30.0],\n",
       " ['Level_96', 30.0, 30.0, 180.0, 30.0],\n",
       " ['Level_97', 30.0, 30.5, 181.0, 30.166666666666668],\n",
       " ['Level_98', 31.0, 31.0, 186.0, 31.0],\n",
       " ['Level_99', 31.0, 31.0, 186.0, 31.0],\n",
       " ['Level_100', 31.0, 32.0, 187.0, 31.166666666666668],\n",
       " ['Level_101', 32.0, 32.0, 192.0, 32.0],\n",
       " ['Level_102', 32.0, 32.0, 192.0, 32.0],\n",
       " ['Level_103', 32.0, 32.5, 192.5, 32.083333333333336],\n",
       " ['Level_104', 32.5, 33.0, 197.5, 32.916666666666664],\n",
       " ['Level_105', 33.0, 33.0, 198.0, 33.0],\n",
       " ['Level_106', 33.0, 34.0, 200.0, 33.333333333333336],\n",
       " ['Level_107', 34.0, 34.0, 204.0, 34.0],\n",
       " ['Level_108', 34.0, 34.0, 204.0, 34.0],\n",
       " ['Level_109', 34.0, 35.0, 208.5, 34.75],\n",
       " ['Level_110', 35.0, 35.0, 210.0, 35.0],\n",
       " ['Level_111', 35.0, 35.0, 210.0, 35.0],\n",
       " ['Level_112', 35.0, 36.0, 214.0, 35.666666666666664],\n",
       " ['Level_113', 36.0, 36.0, 216.0, 36.0],\n",
       " ['Level_114', 36.0, 36.0, 216.0, 36.0],\n",
       " ['Level_115', 36.0, 36.0, 216.0, 36.0],\n",
       " ['Level_116', 36.5, 37.0, 221.5, 36.916666666666664],\n",
       " ['Level_117', 37.0, 38.0, 227.0, 37.833333333333336],\n",
       " ['Level_118', 38.0, 38.0, 228.0, 38.0],\n",
       " ['Level_119', 39.0, 39.0, 234.0, 39.0],\n",
       " ['Level_120', 39.0, 39.0, 234.0, 39.0],\n",
       " ['Level_121', 39.0, 40.0, 238.0, 39.666666666666664],\n",
       " ['Level_122', 40.0, 40.0, 240.0, 40.0],\n",
       " ['Level_123', 40.0, 41.0, 242.0, 40.333333333333336],\n",
       " ['Level_124', 41.0, 42.0, 247.0, 41.166666666666664],\n",
       " ['Level_125', 42.0, 42.0, 252.0, 42.0],\n",
       " ['Level_126', 42.0, 42.0, 252.0, 42.0],\n",
       " ['Level_127', 43.0, 44.0, 259.0, 43.166666666666664],\n",
       " ['Level_128', 44.0, 44.0, 264.0, 44.0],\n",
       " ['Level_129', 44.0, 45.0, 268.0, 44.666666666666664],\n",
       " ['Level_130', 45.0, 45.0, 270.0, 45.0],\n",
       " ['Level_131', 45.0, 46.0, 273.0, 45.5],\n",
       " ['Level_132', 46.0, 47.0, 281.0, 46.833333333333336],\n",
       " ['Level_133', 47.0, 48.0, 284.0, 47.333333333333336],\n",
       " ['Level_134', 48.0, 48.0, 288.0, 48.0],\n",
       " ['Level_135', 48.0, 49.0, 293.0, 48.833333333333336],\n",
       " ['Level_136', 49.0, 50.0, 299.0, 49.833333333333336],\n",
       " ['Level_137', 50.0, 51.0, 301.0, 50.166666666666664],\n",
       " ['Level_138', 51.0, 51.0, 306.0, 51.0],\n",
       " ['Level_139', 52.0, 52.0, 312.0, 52.0],\n",
       " ['Level_140', 53.0, 54.0, 323.0, 53.833333333333336],\n",
       " ['Level_141', 54.0, 55.5, 327.5, 54.583333333333336],\n",
       " ['Level_142', 56.0, 57.0, 338.0, 56.333333333333336],\n",
       " ['Level_143', 58.0, 59.0, 349.0, 58.166666666666664],\n",
       " ['Level_144', 59.0, 61.0, 360.0, 60.0],\n",
       " ['Level_145', 61.0, 62.0, 370.0, 61.666666666666664],\n",
       " ['Level_146', 63.0, 65.0, 384.0, 64.0],\n",
       " ['Level_147', 65.0, 71.0, 412.5, 68.75],\n",
       " ['Level_148', 71.0, 80.0, 225.0, 75.0]]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_hierarchy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that: we divide Age attribute into bins, bin_depth=6, for each tuple, we have level, bin_min, bin_max, bin_sum, bin_mean"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}