aihub_json_preprocessing_general_common_sense.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AIHub Json Preprocessing"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Development Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install kss==3.7.3"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "KSS Argument Error: Restart Jupyter Kernel Runtime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install python-mecab-ko"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "KSS 3.7.3 matches python-mecab-ko"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install ray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import os\n",
    "import kss\n",
    "import ray\n",
    "import json\n",
    "import time\n",
    "from time import sleep\n",
    "from tqdm import tqdm\n",
    "from mecab import MeCab\n",
    "import pandas as pd\n",
    "from glob import glob\n",
    "from itertools import chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'c:\\\\Users\\\\MinSeok\\\\Documents\\\\AIHUB'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pwd"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AIHUB 일반상식"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Source](https://aihub.or.kr/aihubdata/data/view.do?currMenu=116&topMenu=100&aihubDataSe=ty&dataSetSn=106)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert JSON File to TXT File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from data_preprocessing import make_json_txt_file_path_list\n",
    "from data_preprocessing import count_number_of_txt_file_with_batch_list\n",
    "from data_preprocessing import write_jsontext_to_txt_file_with_batch_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "json_path_list = ['AIHUB_일반상식/'+ '/**/*.json']\n",
    "txt_path_list = [\"exploration/general_common_sense_pro/AIHUB_general_common_sense_\"]\n",
    "corpus_name = \"general_common_sense\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The number of file: 125\n"
     ]
    }
   ],
   "source": [
    "json_file_list, txt_file_path_list = \\\n",
    "    make_json_txt_file_path_list(json_path_list, txt_path_list, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 10000\n",
      "The number of txt file: 131\n"
     ]
    }
   ],
   "source": [
    "batch_size = 10000\n",
    "corpus_name = 'general_common_sense'\n",
    "the_number_of_txt_file, the_number_of_txt_file_list = count_number_of_txt_file_with_batch_list(json_file_list, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>File</th>\n",
       "      <th>Length of Source List</th>\n",
       "      <th>The Number of txt File</th>\n",
       "      <th>Description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>AIHUB_일반상식\\ko_wiki_v1_squad.json  ~  AIHUB_일반상...</td>\n",
       "      <td>68538</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>AIHUB_일반상식\\mutual_무형대용어_상호참조복원\\edited_[A1] 000...</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>not subject of batch. small source list.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>AIHUB_일반상식\\mutual_무형대용어_상호참조복원\\edited_[A1] 006...</td>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>not subject of batch. small source list.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>AIHUB_일반상식\\mutual_무형대용어_상호참조복원\\edited_[A1] 012...</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>not subject of batch. small source list.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>AIHUB_일반상식\\mutual_무형대용어_상호참조복원\\edited_[A2] 000...</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>not subject of batch. small source list.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>120</td>\n",
       "      <td>AIHUB_일반상식\\mutual_무형대용어_상호참조복원\\edited_[A3] 000...</td>\n",
       "      <td>51</td>\n",
       "      <td>1</td>\n",
       "      <td>not subject of batch. small source list.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>121</td>\n",
       "      <td>AIHUB_일반상식\\mutual_무형대용어_상호참조복원\\edited_[A1] 009...</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>not subject of batch. small source list.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>122</td>\n",
       "      <td>AIHUB_일반상식\\mutual_무형대용어_상호참조복원\\edited_[A1] 032...</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>not subject of batch. small source list.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>123</td>\n",
       "      <td>AIHUB_일반상식\\mutual_무형대용어_상호참조복원\\edited_[A1] 010...</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>not subject of batch. small source list.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>124</td>\n",
       "      <td>AIHUB_일반상식\\mutual_무형대용어_상호참조복원\\edited_[A1] 013...</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>not subject of batch. small source list.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>125 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Unnamed: 0                                               File  \\\n",
       "0             0  AIHUB_일반상식\\ko_wiki_v1_squad.json  ~  AIHUB_일반상...   \n",
       "1             1  AIHUB_일반상식\\mutual_무형대용어_상호참조복원\\edited_[A1] 000...   \n",
       "2             2  AIHUB_일반상식\\mutual_무형대용어_상호참조복원\\edited_[A1] 006...   \n",
       "3             3  AIHUB_일반상식\\mutual_무형대용어_상호참조복원\\edited_[A1] 012...   \n",
       "4             4  AIHUB_일반상식\\mutual_무형대용어_상호참조복원\\edited_[A2] 000...   \n",
       "..          ...                                                ...   \n",
       "120         120  AIHUB_일반상식\\mutual_무형대용어_상호참조복원\\edited_[A3] 000...   \n",
       "121         121  AIHUB_일반상식\\mutual_무형대용어_상호참조복원\\edited_[A1] 009...   \n",
       "122         122  AIHUB_일반상식\\mutual_무형대용어_상호참조복원\\edited_[A1] 032...   \n",
       "123         123  AIHUB_일반상식\\mutual_무형대용어_상호참조복원\\edited_[A1] 010...   \n",
       "124         124  AIHUB_일반상식\\mutual_무형대용어_상호참조복원\\edited_[A1] 013...   \n",
       "\n",
       "     Length of Source List  The Number of txt File  \\\n",
       "0                    68538                       7   \n",
       "1                       26                       1   \n",
       "2                       37                       1   \n",
       "3                        6                       1   \n",
       "4                       50                       1   \n",
       "..                     ...                     ...   \n",
       "120                     51                       1   \n",
       "121                      9                       1   \n",
       "122                      8                       1   \n",
       "123                      7                       1   \n",
       "124                     25                       1   \n",
       "\n",
       "                                  Description  \n",
       "0                                         NaN  \n",
       "1    not subject of batch. small source list.  \n",
       "2    not subject of batch. small source list.  \n",
       "3    not subject of batch. small source list.  \n",
       "4    not subject of batch. small source list.  \n",
       "..                                        ...  \n",
       "120  not subject of batch. small source list.  \n",
       "121  not subject of batch. small source list.  \n",
       "122  not subject of batch. small source list.  \n",
       "123  not subject of batch. small source list.  \n",
       "124  not subject of batch. small source list.  \n",
       "\n",
       "[125 rows x 5 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "source_file_by_batch_df = pd.read_excel('source_file_by_batch/general_common_sense.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Size]\n",
      "The number of preprocessing corpus: 131\n",
      "\n",
      "[Order]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/131 [03:24<?, ?it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 2 3 4 5 6 "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7 "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▍| 124/131 [00:14<00:00,  8.45it/s]A\n"
     ]
    }
   ],
   "source": [
    "batch_size = 10000\n",
    "corpus_name = \"general_common_sense\"\n",
    "write_jsontext_to_txt_file_with_batch_list(json_file_list, txt_file_path_list, batch_size, the_number_of_txt_file_list, corpus_name)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess TXT File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_segmentation import preprocessing_text\n",
    "from data_preprocessing import make_pro_post_txt_file_path_list\n",
    "from data_preprocessing import merge_and_deduplicate_corpus_txt\n",
    "from reading_data import reading_txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pro_corpus_path = \"AIHUB_corpus/exploration/general_common_sense_pro/AIHUB_general_common_sense_\" + \"*.txt\"\n",
    "pro_post_xlsx_path = \"pro_post_corpus_path/general_common_sense.xlsx\"\n",
    "pro_total_corpus_path_list, post_total_corpus_path_list = make_pro_post_txt_file_path_list(pro_corpus_path, pro_post_xlsx_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(pro_total_corpus_path_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pro_coprus_file = pro_total_corpus_path_list[0]\n",
    "line_length = 1\n",
    "data_type = \"source\"\n",
    "\n",
    "reading_txt(pro_coprus_file, line_length, data_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pro_coprus_file = pro_total_corpus_path_list[0]\n",
    "line_length = 1\n",
    "data_type = \"preprocessing\"\n",
    "\n",
    "reading_txt(pro_coprus_file, line_length, data_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/general_common_sense.xlsx' \\\n",
    "    --start_index 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ray.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessing_corpus_path = \"AIHUB_corpus/exploration/general_common_sense/AIHUB_general_common_sense_\" +\"*.txt\"\n",
    "merge_corpus_path = 'AIHUB_corpus/duplicate/AIHUB_general_common_sense.txt'\n",
    "deduplicate_corpus_path = 'AIHUB_corpus/AIHUB_general_common_sense.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "merge_and_deduplicate_corpus_txt(preprocessing_corpus_path, merge_corpus_path, \n",
    "                                  deduplicate_corpus_path)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "corpus_05",
   "language": "python",
   "name": "corpus_05"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}