aihub_json_preprocessing_news_article_machine_reading_data.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AIHub Json Preprocessing"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Development Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install kss==3.7.3"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "KSS Argument Error: Restart Jupyter Kernel Runtime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install python-mecab-ko"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "KSS 3.7.3 matches python-mecab-ko"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install ray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import os\n",
    "import kss\n",
    "import ray\n",
    "import json\n",
    "import time\n",
    "from time import sleep\n",
    "from tqdm import tqdm\n",
    "from mecab import MeCab\n",
    "import pandas as pd\n",
    "from glob import glob\n",
    "from itertools import chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'c:\\\\Users\\\\MinSeok\\\\Documents\\\\AIHUB'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pwd"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AIHUB 뉴스기사 기계독해 데이터"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Source](https://aihub.or.kr/aihubdata/data/view.do?currMenu=115&topMenu=100&aihubDataSe=realm&dataSetSn=577)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert JSON File to TXT File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from data_preprocessing import make_train_valid_json_txt_file_path_list\n",
    "from data_preprocessing import count_number_of_txt_file_with_batch_list\n",
    "from data_preprocessing import write_jsontext_to_txt_file_with_batch_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "json_path_list = ['AIHUB_뉴스 기사 기계독해 데이터/Training/'+ '/**/*.json', \n",
    "                  'AIHUB_뉴스 기사 기계독해 데이터/Validation/'+ '/**/*.json']\n",
    "txt_path_list = [\"exploration/news_article_machine_reading_data_pro/AIHUB_news_article_machine_reading_data_train_\", \n",
    "                 \"exploration/news_article_machine_reading_data_pro/AIHUB_news_article_machine_reading_data_valid_\"]\n",
    "corpus_name = \"news_article_machine_reading_data\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The number of file: 8\n"
     ]
    }
   ],
   "source": [
    "train_json_file_list, valid_json_file_list, train_txt_file_path_list, valid_txt_file_path_list = \\\n",
    "    make_train_valid_json_txt_file_path_list(json_path_list, txt_path_list, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 324\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = 'news_article_machine_reading_data'\n",
    "the_number_of_train_txt_file, the_number_of_train_txt_file_list = count_number_of_txt_file_with_batch_list(train_json_file_list, batch_size, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>File</th>\n",
       "      <th>Length of Source List</th>\n",
       "      <th>The Number of txt File</th>\n",
       "      <th>Description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>AIHUB_뉴스 기사 기계독해 데이터/Training\\원천데이터\\TS_unanswe...</td>\n",
       "      <td>16000</td>\n",
       "      <td>17</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>AIHUB_뉴스 기사 기계독해 데이터/Training\\원천데이터\\TS_span_in...</td>\n",
       "      <td>32012</td>\n",
       "      <td>33</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>AIHUB_뉴스 기사 기계독해 데이터/Training\\원천데이터\\TS_span_ex...</td>\n",
       "      <td>224020</td>\n",
       "      <td>225</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>AIHUB_뉴스 기사 기계독해 데이터/Training\\원천데이터\\TS_text_en...</td>\n",
       "      <td>48022</td>\n",
       "      <td>49</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                               File  \\\n",
       "0           0  AIHUB_뉴스 기사 기계독해 데이터/Training\\원천데이터\\TS_unanswe...   \n",
       "1           1  AIHUB_뉴스 기사 기계독해 데이터/Training\\원천데이터\\TS_span_in...   \n",
       "2           2  AIHUB_뉴스 기사 기계독해 데이터/Training\\원천데이터\\TS_span_ex...   \n",
       "3           3  AIHUB_뉴스 기사 기계독해 데이터/Training\\원천데이터\\TS_text_en...   \n",
       "\n",
       "   Length of Source List  The Number of txt File  Description  \n",
       "0                  16000                      17          NaN  \n",
       "1                  32012                      33          NaN  \n",
       "2                 224020                     225          NaN  \n",
       "3                  48022                      49          NaN  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "source_file_by_batch_train_df = pd.read_excel('source_file_by_batch/news_article_machine_reading_data_train.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Size]\n",
      "The number of preprocessing corpus: 324\n",
      "\n",
      "[Order]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████▉| 323/324 [00:26<00:00, 12.31it/s]\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = 'news_article_machine_reading_data'\n",
    "write_jsontext_to_txt_file_with_batch_list(train_json_file_list, train_txt_file_path_list, batch_size, the_number_of_train_txt_file_list, corpus_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch Size: 1000\n",
      "The number of txt file: 44\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "corpus_name = 'news_article_machine_reading_data'\n",
    "the_number_of_valid_txt_file, the_number_of_valid_txt_file_list = count_number_of_txt_file_with_batch_list(valid_json_file_list, batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>File</th>\n",
       "      <th>Length of Source List</th>\n",
       "      <th>The Number of txt File</th>\n",
       "      <th>Description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>AIHUB_뉴스 기사 기계독해 데이터/Validation\\원천데이터\\VS_unans...</td>\n",
       "      <td>2000</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>AIHUB_뉴스 기사 기계독해 데이터/Validation\\원천데이터\\VS_span_...</td>\n",
       "      <td>4002</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>AIHUB_뉴스 기사 기계독해 데이터/Validation\\원천데이터\\VS_span_...</td>\n",
       "      <td>28002</td>\n",
       "      <td>29</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>AIHUB_뉴스 기사 기계독해 데이터/Validation\\원천데이터\\VS_text_...</td>\n",
       "      <td>6002</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                               File  \\\n",
       "0           0  AIHUB_뉴스 기사 기계독해 데이터/Validation\\원천데이터\\VS_unans...   \n",
       "1           1  AIHUB_뉴스 기사 기계독해 데이터/Validation\\원천데이터\\VS_span_...   \n",
       "2           2  AIHUB_뉴스 기사 기계독해 데이터/Validation\\원천데이터\\VS_span_...   \n",
       "3           3  AIHUB_뉴스 기사 기계독해 데이터/Validation\\원천데이터\\VS_text_...   \n",
       "\n",
       "   Length of Source List  The Number of txt File  Description  \n",
       "0                   2000                       3          NaN  \n",
       "1                   4002                       5          NaN  \n",
       "2                  28002                      29          NaN  \n",
       "3                   6002                       7          NaN  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "source_file_by_batch_valid_df = pd.read_excel('source_file_by_batch/news_article_machine_reading_data_valid.xlsx', engine='openpyxl')  \n",
    "source_file_by_batch_valid_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Size]\n",
      "The number of preprocessing corpus: 44\n",
      "\n",
      "[Order]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|█▎        | 6/44 [00:02<00:15,  2.43it/s]\n"
     ]
    }
   ],
   "source": [
    "batch_size = 10000\n",
    "corpus_name = 'news_article_machine_reading_data'\n",
    "write_jsontext_to_txt_file_with_batch_list(valid_json_file_list, valid_txt_file_path_list, batch_size, the_number_of_valid_txt_file_list, corpus_name)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess TXT File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_segmentation import preprocessing_text\n",
    "from data_preprocessing import make_pro_post_txt_file_path_list\n",
    "from data_preprocessing import merge_and_deduplicate_corpus_txt\n",
    "from reading_data import reading_txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pro_corpus_path = \"AIHUB_corpus/demo/news_article_machine_reading_data_pro/AIHUB_news_article_machine_reading_data_\" + \"*.txt\"\n",
    "pro_post_xlsx_path = \"pro_post_corpus_path/news_article_machine_reading_data.xlsx\"\n",
    "pro_total_corpus_path_list, post_total_corpus_path_list = make_pro_post_txt_file_path_list(pro_corpus_path, pro_post_xlsx_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(pro_total_corpus_path_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pro_coprus_file = pro_total_corpus_path_list[0]\n",
    "line_length = 1\n",
    "data_type = \"source\"\n",
    "\n",
    "reading_txt(pro_coprus_file, line_length, data_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pro_coprus_file = pro_total_corpus_path_list[0]\n",
    "line_length = 1\n",
    "data_type = \"preprocessing\"\n",
    "\n",
    "reading_txt(pro_coprus_file, line_length, data_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m txt_preprocessing \\\n",
    "    --num_cpus 4 \\\n",
    "    --pro_post_xlsx_path 'pro_post_corpus_path/news_article_machine_reading_data.xlsx' \\\n",
    "    --start_index 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ray.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessing_corpus_path = \"AIHUB_corpus/exploration/news_article_machine_reading_data_post/AIHUB_news_article_machine_reading_data_\" +\"*.txt\"\n",
    "merge_corpus_path = 'AIHUB_corpus/duplicate/AIHUB_news_article_machine_reading_data.txt'\n",
    "deduplicate_corpus_path = 'AIHUB_corpus/AIHUB_news_article_machine_reading_data.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "merge_and_deduplicate_corpus_list(preprocessing_corpus_path, merge_corpus_path, \n",
    "                                  deduplicate_corpus_path)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "corpus_06",
   "language": "python",
   "name": "corpus_06"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}